Merge branch '3.0' into test/jcy

Jenkinsfile2

@@ -88,6 +88,12 @@ def pre_test(){
     cmake .. > /dev/null
     make -j4> /dev/null
     '''
+    sh'''
+    cd ${WKPY}
+    git reset --hard
+    git pull
+    pip3 install .
+    '''
     return 1
 }
 
@@ -97,6 +103,7 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
+      WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
   }
   stages {
       stage('pre_build'){
@@ -117,6 +124,11 @@ pipeline {
               ./test-all.sh b1fq
               '''
               sh'''
+              export LD_LIBRARY_PATH=${WKC}/debug/build/lib
+              cd ${WKC}/tests/system-test
+              ./fulltest.sh
+              '''
+              sh'''
               cd ${WKC}/debug
               ctest
               '''

example/src/demoapi.c

@@ -176,7 +176,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
     }
 
     if (block) {
-        warnPrint("%s", "call taos_fetch_block()\n");
+        warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
         int rows = 0;
         while ((rows = taos_fetch_block(res, &row))) {
             int *lengths = taos_fetch_lengths(res);

example/src/tmq.c

@@ -22,6 +22,7 @@
 static int  running = 1;
 static void msg_process(TAOS_RES* msg) {
   char buf[1024];
+  memset(buf, 0, 1024);
   printf("topic: %s\n", tmq_get_topic_name(msg));
   printf("vg:%d\n", tmq_get_vgroup_id(msg));
   while (1) {
@@ -140,7 +141,7 @@ int32_t create_topic() {
   return 0;
 }
 
-void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
+void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) {
   printf("commit %d\n", resp);
 }
 
@@ -162,7 +163,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
   tmq_conf_set(conf, "td.connect.db", "abc1");
   tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
-  tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
+  tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
   return tmq;
 }
 
@@ -188,7 +189,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
       cnt++;
       /*printf("get data\n");*/
       /*msg_process(tmqmessage);*/
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
       /*} else {*/
       /*break;*/
     }
@@ -218,9 +219,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000);
     if (tmqmessage) {
       msg_process(tmqmessage);
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
 
-      if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
+      /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/
     }
   }
 
@@ -248,7 +249,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
       batchCnt++;
       /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/
       /*msg_process(tmqmessage);*/
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
     } else {
       break;
     }

include/client/taos.h

@@ -92,38 +92,14 @@ typedef struct taosField {
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
-typedef struct TAOS_BIND {
+typedef struct TAOS_BIND_v2 {
-  int        buffer_type;
-  void      *buffer;
-  uintptr_t  buffer_length;  // unused
-  uintptr_t *length;
-  int       *is_null;
-
-  int  is_unsigned;  // unused
-  int *error;        // unused
-  union {
-    int64_t        ts;
-    int8_t         b;
-    int8_t         v1;
-    int16_t        v2;
-    int32_t        v4;
-    int64_t        v8;
-    float          f4;
-    double         f8;
-    unsigned char *bin;
-    char          *nchar;
-  } u;
-  unsigned int allocated;
-} TAOS_BIND;
-
-typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
   void     *buffer;
-  uintptr_t buffer_length;
+  int32_t   buffer_length;
   int32_t  *length;
   char     *is_null;
   int       num;
-} TAOS_MULTI_BIND;
+} TAOS_BIND_v2;
 
 typedef enum {
   SET_CONF_RET_SUCC = 0,
@@ -152,34 +128,34 @@ DLL_EXPORT void  taos_close(TAOS *taos);
 
 const char *taos_data_type(int type);
 
-DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
+DLL_EXPORT TAOS_STMT  *taos_stmt_init(TAOS *taos);
-DLL_EXPORT int        taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
+DLL_EXPORT int         taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags);
+DLL_EXPORT int         taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags);
-DLL_EXPORT int        taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
+DLL_EXPORT int         taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
-DLL_EXPORT int        taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
+DLL_EXPORT int         taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
 
-DLL_EXPORT int       taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
+DLL_EXPORT int         taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
-DLL_EXPORT int       taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
+DLL_EXPORT int         taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
-DLL_EXPORT int       taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
+DLL_EXPORT int         taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind);
+DLL_EXPORT int         taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
-DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int         taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
-DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
+DLL_EXPORT int         taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx);
-DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
+DLL_EXPORT int         taos_stmt_add_batch(TAOS_STMT *stmt);
-DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
+DLL_EXPORT int         taos_stmt_execute(TAOS_STMT *stmt);
-DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
+DLL_EXPORT TAOS_RES   *taos_stmt_use_result(TAOS_STMT *stmt);
-DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
+DLL_EXPORT int         taos_stmt_close(TAOS_STMT *stmt);
-DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT char       *taos_stmt_errstr(TAOS_STMT *stmt);
-DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
+DLL_EXPORT int         taos_stmt_affected_rows(TAOS_STMT *stmt);
 
-DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
+DLL_EXPORT TAOS_RES   *taos_query(TAOS *taos, const char *sql);
-DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
+DLL_EXPORT TAOS_RES   *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
 
-DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
+DLL_EXPORT TAOS_ROW    taos_fetch_row(TAOS_RES *res);
-DLL_EXPORT int      taos_result_precision(TAOS_RES *res);  // get the time precision of result
+DLL_EXPORT int         taos_result_precision(TAOS_RES *res);  // get the time precision of result
-DLL_EXPORT void     taos_free_result(TAOS_RES *res);
+DLL_EXPORT void        taos_free_result(TAOS_RES *res);
-DLL_EXPORT int      taos_field_count(TAOS_RES *res);
+DLL_EXPORT int         taos_field_count(TAOS_RES *res);
-DLL_EXPORT int      taos_num_fields(TAOS_RES *res);
+DLL_EXPORT int         taos_num_fields(TAOS_RES *res);
-DLL_EXPORT int      taos_affected_rows(TAOS_RES *res);
+DLL_EXPORT int         taos_affected_rows(TAOS_RES *res);
 
 DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
 DLL_EXPORT int         taos_select_db(TAOS *taos, const char *db);
@@ -241,17 +217,17 @@ typedef struct tmq_conf_t tmq_conf_t;
 typedef struct tmq_list_t tmq_list_t;
 // typedef struct tmq_message_t tmq_message_t;
 
-typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param));
+typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *));
 
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 
-#if 1
+#if 0
 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 #endif
 
-DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
@@ -295,14 +271,19 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
 
 DLL_EXPORT char   *tmq_get_topic_name(TAOS_RES *res);
 DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
+// TODO
+#if 0
+DLL_EXPORT char   *tmq_get_block_table_name(TAOS_RES *res);
+#endif
+
 #if 0
 DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
 DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
 DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
-#endif
 DLL_EXPORT void tmq_message_destroy(TAOS_RES *res);
+#endif
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
 #if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);

include/common/tdatablock.h

@@ -54,12 +54,34 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
+#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
+#define colDataSetNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] = -1)
+
+#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
+
+#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
+
+#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
+// SColumnInfoData, rowNumber
+#define colDataGetData(p1_, r_)                                        \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
+                                        :  colDataGetNumData(p1_, r_))
+
 static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
+  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){
+    if(colDataIsNull_var(pColumnInfoData, row)){
+      return true;
+    }
+    char *data = colDataGetVarData(pColumnInfoData, row);
+    return (*data == TSDB_DATA_TYPE_NULL);
+  } 
+  
   if (!pColumnInfoData->hasNull) {
     return false;
   }
-  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+
-    return pColumnInfoData->varmeta.offset[row] == -1;
+  if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
+    return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
       return false;
@@ -86,7 +108,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    return pColumnInfoData->varmeta.offset[row] == -1;
+    return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
       return false;
@@ -96,17 +118,10 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 }
 
-#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
-
-// SColumnInfoData, rowNumber
-#define colDataGetData(p1_, r_)                                                        \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
-                                        : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
-
 static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
   // There is a placehold for each NULL value of binary or nchar type.
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    pColumnInfoData->varmeta.offset[currentRow] = -1;  // it is a null value of VAR type.
+    colDataSetNull_var(pColumnInfoData, currentRow);  // it is a null value of VAR type.
   } else {
     colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
   }
@@ -117,7 +132,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
 static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     for (int32_t i = start; i < start + nRows; ++i) {
-      pColumnInfoData->varmeta.offset[i] = -1;  // it is a null value of VAR type.
+      colDataSetNull_var(pColumnInfoData,i);  // it is a null value of VAR type.
     }
   } else {
     for (int32_t i = start; i < start + nRows; ++i) {
@@ -265,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
 #endif
 
 #endif /*_TD_COMMON_EP_H_*/
+

include/common/tdataformat.h

@@ -502,7 +502,7 @@ typedef struct {
 
 #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
-#define kvRowLen(r)            (*(TDRowLenT *)(r))
+#define kvRowLen(r)            (*(uint16_t *)(r))
 #define kvRowNCols(r)          (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
 #define kvRowSetLen(r, len)    kvRowLen(r) = (len)
 #define kvRowSetNCols(r, n)    kvRowNCols(r) = (n)
@@ -608,7 +608,7 @@ void    tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void    tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow  tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) {
+static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
@@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
 
   pBuilder->nCols++;
 
-  int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
   if (tlen > pBuilder->alloc - pBuilder->size) {
     while (tlen > pBuilder->alloc - pBuilder->size) {
       pBuilder->alloc *= 2;

include/common/tglobal.h

@@ -32,7 +32,6 @@ extern char     tsLocalEp[];
 extern uint16_t tsServerPort;
 extern int32_t  tsVersion;
 extern int32_t  tsStatusInterval;
-extern bool     tsEnableTelemetryReporting;
 
 // common
 extern int32_t tsRpcTimer;
@@ -82,6 +81,12 @@ extern uint16_t tsMonitorPort;
 extern int32_t  tsMonitorMaxLogs;
 extern bool     tsMonitorComp;
 
+// telem
+extern bool     tsEnableTelem;
+extern int32_t  tsTelemInterval;
+extern char     tsTelemServer[];
+extern uint16_t tsTelemPort;
+
 // query buffer management
 extern int32_t tsQueryBufferSize;  // maximum allowed usage buffer size in MB for each data node during query processing
 extern int64_t tsQueryBufferSizeBytes;   // maximum allowed usage buffer size in byte for each data node

include/common/tmsg.h

@@ -269,6 +269,8 @@ typedef struct SSchema {
 #define SSCHMEA_BYTES(s) ((s)->bytes)
 #define SSCHMEA_NAME(s)  ((s)->name)
 
+STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
+
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
@@ -322,13 +324,15 @@ typedef struct SEpSet {
 int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp);
 int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
 int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
-void*   taosDecodeSEpSet(void* buf, SEpSet* pEp);
+void*   taosDecodeSEpSet(const void* buf, SEpSet* pEp);
 
 typedef struct {
   int8_t  connType;
   int32_t pid;
   char    app[TSDB_APP_NAME_LEN];
   char    db[TSDB_DB_NAME_LEN];
+  char    user[TSDB_USER_LEN];
+  char    passwd[TSDB_PASSWORD_LEN];
   int64_t startTime;
 } SConnectReq;
 
@@ -480,7 +484,7 @@ typedef struct {
   char    intervalUnit;
   char    slidingUnit;
   char
-      offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
+          offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
   int8_t  precision;
   int64_t interval;
   int64_t sliding;
@@ -658,6 +662,7 @@ typedef struct {
   int8_t  outputType;
   int32_t outputLen;
   int32_t bufSize;
+  int32_t codeLen;
   int64_t signature;
   char*   pComment;
   char*   pCode;
@@ -1270,11 +1275,16 @@ typedef struct {
 } SMVCreateStreamRsp, SMSCreateStreamRsp;
 
 typedef struct {
-  char   name[TSDB_TOPIC_FNAME_LEN];
+  char    name[TSDB_TOPIC_FNAME_LEN];
-  int8_t igExists;
+  int8_t  igExists;
-  char*  sql;
+  int8_t  withTbName;
-  char*  ast;
+  int8_t  withSchema;
-  char   subscribeDbName[TSDB_DB_NAME_LEN];
+  int8_t  withTag;
+  int8_t  withTagSchema;
+  char*   sql;
+  char*   ast;
+  int64_t subDbUid;
+  char    subscribeDbName[TSDB_DB_NAME_LEN];
 } SCMCreateTopicReq;
 
 int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq);
@@ -1288,10 +1298,14 @@ typedef struct {
 int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
 int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
 
+typedef struct {
+  int64_t consumerId;
+} SMqConsumerLostMsg;
+
 typedef struct {
   int32_t topicNum;
   int64_t consumerId;
-  char*   consumerGroup;
+  char    cgroup[TSDB_CGROUP_LEN];
   SArray* topicNames;  // SArray<char*>
 } SCMSubscribeReq;
 
@@ -1299,7 +1313,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pReq->topicNum);
   tlen += taosEncodeFixedI64(buf, pReq->consumerId);
-  tlen += taosEncodeString(buf, pReq->consumerGroup);
+  tlen += taosEncodeString(buf, pReq->cgroup);
 
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i));
@@ -1310,7 +1324,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
 static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
   buf = taosDecodeFixedI32(buf, &pReq->topicNum);
   buf = taosDecodeFixedI64(buf, &pReq->consumerId);
-  buf = taosDecodeString(buf, &pReq->consumerGroup);
+  buf = taosDecodeStringTo(buf, pReq->cgroup);
   pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*));
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     char* name;
@@ -1386,10 +1400,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq
 }
 
 typedef struct {
-  const char* key;
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
-  SArray*     lostConsumers;     // SArray<int64_t>
+  SArray* lostConsumers;     // SArray<int64_t>
-  SArray*     removedConsumers;  // SArray<int64_t>
+  SArray* removedConsumers;  // SArray<int64_t>
-  SArray*     newConsumers;      // SArray<int64_t>
+  SArray* newConsumers;      // SArray<int64_t>
 } SMqRebSubscribe;
 
 static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) {
@@ -1397,7 +1411,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) {
   if (pRebSub == NULL) {
     goto _err;
   }
-  pRebSub->key = strdup(key);
+  strcpy(pRebSub->key, key);
   pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t));
   if (pRebSub->lostConsumers == NULL) {
     goto _err;
@@ -1422,6 +1436,7 @@ _err:
 // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or
 // deserialization
 typedef struct {
+  int8_t*   mqInReb;
   SHashObj* rebSubHash;  // SHashObj<key, SMqRebSubscribe>
 } SMqDoRebalanceMsg;
 
@@ -1924,6 +1939,64 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
   return buf;
 }
 
+enum {
+  TOPIC_SUB_TYPE__DB = 1,
+  TOPIC_SUB_TYPE__TABLE,
+};
+
+typedef struct {
+  int64_t leftForVer;
+  int32_t vgId;
+  int64_t oldConsumerId;
+  int64_t newConsumerId;
+  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int8_t  subType;
+  int8_t  withTbName;
+  int8_t  withSchema;
+  int8_t  withTag;
+  int8_t  withTagSchema;
+  char*   qmsg;
+} SMqRebVgReq;
+
+static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
+  tlen += taosEncodeFixedI32(buf, pReq->vgId);
+  tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
+  tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
+  tlen += taosEncodeString(buf, pReq->subKey);
+  tlen += taosEncodeFixedI8(buf, pReq->subType);
+  tlen += taosEncodeFixedI8(buf, pReq->withTbName);
+  tlen += taosEncodeFixedI8(buf, pReq->withSchema);
+  tlen += taosEncodeFixedI8(buf, pReq->withTag);
+  tlen += taosEncodeFixedI8(buf, pReq->withTagSchema);
+  if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
+    tlen += taosEncodeString(buf, pReq->qmsg);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) {
+  buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
+  buf = taosDecodeFixedI32(buf, &pReq->vgId);
+  buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
+  buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
+  buf = taosDecodeStringTo(buf, pReq->subKey);
+  buf = taosDecodeFixedI8(buf, &pReq->subType);
+  buf = taosDecodeFixedI8(buf, &pReq->withTbName);
+  buf = taosDecodeFixedI8(buf, &pReq->withSchema);
+  buf = taosDecodeFixedI8(buf, &pReq->withTag);
+  buf = taosDecodeFixedI8(buf, &pReq->withTagSchema);
+  if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
+    buf = taosDecodeString(buf, &pReq->qmsg);
+  }
+  return (void*)buf;
+}
+
+typedef struct {
+  int8_t reserved;
+} SMqRebVgRsp;
+
 typedef struct {
   int64_t leftForVer;
   int32_t vgId;
@@ -2394,6 +2467,7 @@ typedef struct {
   int64_t consumerId;
 } SMqRspHead;
 
+#if 0
 typedef struct {
   SMsgHead head;
 
@@ -2407,6 +2481,17 @@ typedef struct {
   uint64_t reqId;
   char     topic[TSDB_TOPIC_FNAME_LEN];
 } SMqPollReq;
+#endif
+
+typedef struct {
+  SMsgHead head;
+  char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int32_t  epoch;
+  uint64_t reqId;
+  int64_t  consumerId;
+  int64_t  blockingTime;
+  int64_t  currentOffset;
+} SMqPollReqV2;
 
 typedef struct {
   int32_t vgId;
@@ -2480,13 +2565,81 @@ static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRs
   return (void*)buf;
 }
 
+typedef struct {
+  SMqRspHead head;
+  int64_t    reqOffset;
+  int64_t    rspOffset;
+  int32_t    skipLogNum;
+  int32_t    blockNum;
+  int8_t     withTbName;
+  int8_t     withSchema;
+  int8_t     withTag;
+  int8_t     withTagSchema;
+  SArray*    blockDataLen;    // SArray<int32_t>
+  SArray*    blockData;       // SArray<SRetrieveTableRsp*>
+  SArray*    blockTbName;     // SArray<char*>
+  SArray*    blockSchema;     // SArray<SSchemaWrapper>
+  SArray*    blockTags;       // SArray<kvrow>
+  SArray*    blockTagSchema;  // SArray<kvrow>
+} SMqDataBlkRsp;
+
+static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
+  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
+  tlen += taosEncodeFixedI32(buf, pRsp->blockNum);
+  if (pRsp->blockNum != 0) {
+    tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
+    tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTag);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
+      void*   data = taosArrayGetP(pRsp->blockData, i);
+      tlen += taosEncodeFixedI32(buf, bLen);
+      tlen += taosEncodeBinary(buf, data, bLen);
+    }
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) {
+  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
+  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
+  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
+  buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
+  pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  if (pRsp->blockNum != 0) {
+    buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
+    buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTag);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = 0;
+      void*   data = NULL;
+      buf = taosDecodeFixedI32(buf, &bLen);
+      buf = taosDecodeBinary(buf, &data, bLen);
+      taosArrayPush(pRsp->blockDataLen, &bLen);
+      taosArrayPush(pRsp->blockData, &data);
+    }
+  }
+  return (void*)buf;
+}
+
 typedef struct {
   SMqRspHead head;
   char       cgroup[TSDB_CGROUP_LEN];
   SArray*    topics;  // SArray<SMqSubTopicEp>
 } SMqCMGetSubEpRsp;
 
-static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
+static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
+  // taosMemoryFree(pSubTopicEp->schema.pSchema);
+  taosArrayDestroy(pSubTopicEp->vgs);
+}
 
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
   int32_t tlen = 0;

include/common/tmsgdef.h

@@ -146,6 +146,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
@@ -177,6 +178,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)

include/common/trow.h

@@ -219,7 +219,7 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return
 
 #define TD_ROW_OFFSET(p) ((p)->toffset);  // During ParseInsert when without STSchema, how to get the offset for STpRow?
 
-void                        tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap);
+void                        tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap);
 static FORCE_INLINE void    tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); }
 static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType);
 static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType);
@@ -308,8 +308,8 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx,
   // use literal value directly and not use formula to simplify the codes
   switch (nOffset) {
     case 0:
+      *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
       // set the value and clear other partitions for offset 0
-      *pDestByte = (valType << 6);
       // *pDestByte |= (valType << 6);
       break;
     case 1:
@@ -417,8 +417,8 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T
   // use literal value directly and not use formula to simplify the codes
   switch (nOffset) {
     case 0:
+      *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7);
       // set the value and clear other partitions for offset 0
-      *pDestByte = (valType << 7);  
       // *pDestByte |= (valType << 7);
       break;
     case 1:
@@ -684,6 +684,43 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief The invoker is responsible for memory alloc/dealloc.
+ *
+ * @param pBuilder
+ * @param pBuf Output buffer of STSRow
+ */
+static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) {
+  pBuilder->pBuf = (STSRow *)pBuf;
+  if (!pBuilder->pBuf) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0);
+
+  uint32_t len = 0;
+  switch (pBuilder->rowType) {
+    case TD_ROW_TP:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen);
+#endif
+      break;
+    case TD_ROW_KV:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
+#endif
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+
 /**
  * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数
  *
@@ -1107,11 +1144,11 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col
   if (TD_IS_TP_ROW(pIter->pRow)) {
     STColumn *pCol = NULL;
     STSchema *pSchema = pIter->pSchema;
-    while (pIter->colIdx <= pSchema->numOfCols) {
+    while (pIter->colIdx < pSchema->numOfCols) {
       pCol = &pSchema->columns[pIter->colIdx];  // 1st column of schema is primary TS key
       if (colId == pCol->colId) {
         break;
-      } else if (colId < pCol->colId) {
+      } else if (pCol->colId < colId) {
         ++pIter->colIdx;
         continue;
       } else {
@@ -1237,6 +1274,101 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows,
   return result;
 }
 
+static void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
+  if (tdValTypeIsNull(pVal->valType)) {
+    printf("NULL ");
+    return;
+  } else if (tdValTypeIsNone(pVal->valType)) {
+    printf("NONE ");
+    return;
+  }
+  switch (colType) {
+    case TSDB_DATA_TYPE_BOOL:
+      printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true");
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      printf("%" PRIi8 " ", *(int8_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      printf("%" PRIi16 " ", *(int16_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_INT:
+      printf("%" PRIi32 " ", *(int32_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      printf("%" PRIi64 " ", *(int64_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      printf("%f ", *(float *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      printf("%lf ", *(double *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_VARCHAR:
+      printf("VARCHAR ");
+      break;
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      printf("%" PRIi64 " ", *(int64_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_NCHAR:
+      printf("NCHAR ");
+      break;
+    case TSDB_DATA_TYPE_UTINYINT:
+      printf("%" PRIu8 " ", *(uint8_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      printf("%" PRIu16 " ", *(uint16_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      printf("%" PRIu32 " ", *(uint32_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      printf("%" PRIu64 " ", *(uint64_t *)pVal->val);
+      break;
+    case TSDB_DATA_TYPE_JSON:
+      printf("JSON ");
+      break;
+    case TSDB_DATA_TYPE_VARBINARY:
+      printf("VARBIN ");
+      break;
+    case TSDB_DATA_TYPE_DECIMAL:
+      printf("DECIMAL ");
+      break;
+    case TSDB_DATA_TYPE_BLOB:
+      printf("BLOB ");
+      break;
+    case TSDB_DATA_TYPE_MEDIUMBLOB:
+      printf("MedBLOB ");
+      break;
+    // case TSDB_DATA_TYPE_BINARY:
+    //   printf("BINARY ");
+    //   break;
+    case TSDB_DATA_TYPE_MAX:
+      printf("UNDEF ");
+      break;
+    default:
+      printf("UNDEF ");
+      break;
+  }
+}
+
+static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
+  STSRowIter iter = {0};
+  tdSTSRowIterInit(&iter, pSchema);
+  tdSTSRowIterReset(&iter, row);
+  printf(">>>");
+  for (int i = 0; i < pSchema->numOfCols; ++i) {
+    STColumn *stCol = pSchema->columns + i;
+    SCellVal  sVal = {.valType = 255, .val = NULL};
+    if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
+      break;
+    }
+    ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2);
+    tdSCellValPrint(&sVal, stCol->type);
+  }
+  printf("\n");
+}
+
 #ifdef TROW_ORIGIN_HZ
 typedef struct {
   uint32_t nRows;

include/common/ttokendef.h

@@ -148,94 +148,95 @@
 #define TK_VARIABLES                      130
 #define TK_BNODES                         131
 #define TK_SNODES                         132
-#define TK_LIKE                           133
+#define TK_CLUSTER                        133
-#define TK_INDEX                          134
+#define TK_LIKE                           134
-#define TK_FULLTEXT                       135
+#define TK_INDEX                          135
-#define TK_FUNCTION                       136
+#define TK_FULLTEXT                       136
-#define TK_INTERVAL                       137
+#define TK_FUNCTION                       137
-#define TK_TOPIC                          138
+#define TK_INTERVAL                       138
-#define TK_AS                             139
+#define TK_TOPIC                          139
-#define TK_DESC                           140
+#define TK_AS                             140
-#define TK_DESCRIBE                       141
+#define TK_DESC                           141
-#define TK_RESET                          142
+#define TK_DESCRIBE                       142
-#define TK_QUERY                          143
+#define TK_RESET                          143
-#define TK_EXPLAIN                        144
+#define TK_QUERY                          144
-#define TK_ANALYZE                        145
+#define TK_EXPLAIN                        145
-#define TK_VERBOSE                        146
+#define TK_ANALYZE                        146
-#define TK_NK_BOOL                        147
+#define TK_VERBOSE                        147
-#define TK_RATIO                          148
+#define TK_NK_BOOL                        148
-#define TK_COMPACT                        149
+#define TK_RATIO                          149
-#define TK_VNODES                         150
+#define TK_COMPACT                        150
-#define TK_IN                             151
+#define TK_VNODES                         151
-#define TK_OUTPUTTYPE                     152
+#define TK_IN                             152
-#define TK_AGGREGATE                      153
+#define TK_OUTPUTTYPE                     153
-#define TK_BUFSIZE                        154
+#define TK_AGGREGATE                      154
-#define TK_STREAM                         155
+#define TK_BUFSIZE                        155
-#define TK_INTO                           156
+#define TK_STREAM                         156
-#define TK_TRIGGER                        157
+#define TK_INTO                           157
-#define TK_AT_ONCE                        158
+#define TK_TRIGGER                        158
-#define TK_WINDOW_CLOSE                   159
+#define TK_AT_ONCE                        159
-#define TK_WATERMARK                      160
+#define TK_WINDOW_CLOSE                   160
-#define TK_KILL                           161
+#define TK_WATERMARK                      161
-#define TK_CONNECTION                     162
+#define TK_KILL                           162
-#define TK_MERGE                          163
+#define TK_CONNECTION                     163
-#define TK_VGROUP                         164
+#define TK_MERGE                          164
-#define TK_REDISTRIBUTE                   165
+#define TK_VGROUP                         165
-#define TK_SPLIT                          166
+#define TK_REDISTRIBUTE                   166
-#define TK_SYNCDB                         167
+#define TK_SPLIT                          167
-#define TK_NULL                           168
+#define TK_SYNCDB                         168
-#define TK_NK_QUESTION                    169
+#define TK_NULL                           169
-#define TK_NK_ARROW                       170
+#define TK_NK_QUESTION                    170
-#define TK_ROWTS                          171
+#define TK_NK_ARROW                       171
-#define TK_TBNAME                         172
+#define TK_ROWTS                          172
-#define TK_QSTARTTS                       173
+#define TK_TBNAME                         173
-#define TK_QENDTS                         174
+#define TK_QSTARTTS                       174
-#define TK_WSTARTTS                       175
+#define TK_QENDTS                         175
-#define TK_WENDTS                         176
+#define TK_WSTARTTS                       176
-#define TK_WDURATION                      177
+#define TK_WENDTS                         177
-#define TK_CAST                           178
+#define TK_WDURATION                      178
-#define TK_NOW                            179
+#define TK_CAST                           179
-#define TK_TODAY                          180
+#define TK_NOW                            180
-#define TK_TIMEZONE                       181
+#define TK_TODAY                          181
-#define TK_COUNT                          182
+#define TK_TIMEZONE                       182
-#define TK_FIRST                          183
+#define TK_COUNT                          183
-#define TK_LAST                           184
+#define TK_FIRST                          184
-#define TK_LAST_ROW                       185
+#define TK_LAST                           185
-#define TK_BETWEEN                        186
+#define TK_LAST_ROW                       186
-#define TK_IS                             187
+#define TK_BETWEEN                        187
-#define TK_NK_LT                          188
+#define TK_IS                             188
-#define TK_NK_GT                          189
+#define TK_NK_LT                          189
-#define TK_NK_LE                          190
+#define TK_NK_GT                          190
-#define TK_NK_GE                          191
+#define TK_NK_LE                          191
-#define TK_NK_NE                          192
+#define TK_NK_GE                          192
-#define TK_MATCH                          193
+#define TK_NK_NE                          193
-#define TK_NMATCH                         194
+#define TK_MATCH                          194
-#define TK_CONTAINS                       195
+#define TK_NMATCH                         195
-#define TK_JOIN                           196
+#define TK_CONTAINS                       196
-#define TK_INNER                          197
+#define TK_JOIN                           197
-#define TK_SELECT                         198
+#define TK_INNER                          198
-#define TK_DISTINCT                       199
+#define TK_SELECT                         199
-#define TK_WHERE                          200
+#define TK_DISTINCT                       200
-#define TK_PARTITION                      201
+#define TK_WHERE                          201
-#define TK_BY                             202
+#define TK_PARTITION                      202
-#define TK_SESSION                        203
+#define TK_BY                             203
-#define TK_STATE_WINDOW                   204
+#define TK_SESSION                        204
-#define TK_SLIDING                        205
+#define TK_STATE_WINDOW                   205
-#define TK_FILL                           206
+#define TK_SLIDING                        206
-#define TK_VALUE                          207
+#define TK_FILL                           207
-#define TK_NONE                           208
+#define TK_VALUE                          208
-#define TK_PREV                           209
+#define TK_NONE                           209
-#define TK_LINEAR                         210
+#define TK_PREV                           210
-#define TK_NEXT                           211
+#define TK_LINEAR                         211
-#define TK_GROUP                          212
+#define TK_NEXT                           212
-#define TK_HAVING                         213
+#define TK_GROUP                          213
-#define TK_ORDER                          214
+#define TK_HAVING                         214
-#define TK_SLIMIT                         215
+#define TK_ORDER                          215
-#define TK_SOFFSET                        216
+#define TK_SLIMIT                         216
-#define TK_LIMIT                          217
+#define TK_SOFFSET                        217
-#define TK_OFFSET                         218
+#define TK_LIMIT                          218
-#define TK_ASC                            219
+#define TK_OFFSET                         219
-#define TK_NULLS                          220
+#define TK_ASC                            220
+#define TK_NULLS                          221
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301

include/common/ttypes.h

@@ -49,7 +49,7 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
+#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor {
                      int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 
-extern tDataTypeDescriptor tDataTypes[15];
+extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
 
 bool isValidDataType(int32_t type);
 

include/dnode/mnode/sdb/sdb.h

@@ -92,7 +92,13 @@ extern "C" {
 typedef struct SMnode  SMnode;
 typedef struct SSdbRaw SSdbRaw;
 typedef struct SSdbRow SSdbRow;
-typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType;
+
+typedef enum {
+  SDB_KEY_BINARY = 1,
+  SDB_KEY_INT32 = 2,
+  SDB_KEY_INT64 = 3,
+} EKeyType;
+
 typedef enum {
   SDB_STATUS_INIT = 0,
   SDB_STATUS_CREATING = 1,

include/libs/function/functionMgt.h

@@ -110,7 +110,10 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_QENDTS,
   FUNCTION_TYPE_WSTARTTS,
   FUNCTION_TYPE_WENDTS,
-  FUNCTION_TYPE_WDURATION
+  FUNCTION_TYPE_WDURATION,
+
+  // user defined funcion
+  FUNCTION_TYPE_UDF = 10000
 } EFunctionType;
 
 struct SqlFunctionCtx;
@@ -138,6 +141,7 @@ bool fmIsWindowClauseFunc(int32_t funcId);
 bool fmIsSpecialDataRequiredFunc(int32_t funcId);
 bool fmIsDynamicScanOptimizedFunc(int32_t funcId);
 bool fmIsMultiResFunc(int32_t funcId);
+bool fmIsUserDefinedFunc(int32_t funcId);
 
 typedef enum EFuncDataRequired {
   FUNC_DATA_REQUIRED_DATA_LOAD = 1,

include/libs/nodes/cmdnodes.h

@@ -295,6 +295,16 @@ typedef struct SDropStreamStmt {
   bool ignoreNotExists;
 } SDropStreamStmt;
 
+typedef struct SCreateFunctionStmt {
+  ENodeType type;
+  bool ignoreExists;
+  char funcName[TSDB_FUNC_NAME_LEN];
+  bool isAgg;
+  char libraryPath[PATH_MAX];
+  SDataType outputDt;
+  int32_t bufSize;
+} SCreateFunctionStmt;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/nodes.h

@@ -48,6 +48,9 @@ extern "C" {
     (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \
     cell1 = cell1->pNext, cell2 = cell2->pNext)
 
+#define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode)
+#define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode)
+
 #define FOREACH_FOR_REWRITE(node, list)	\
   for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
 
@@ -134,6 +137,7 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_QNODES_STMT,
   QUERY_NODE_SHOW_SNODES_STMT,
   QUERY_NODE_SHOW_BNODES_STMT,
+  QUERY_NODE_SHOW_CLUSTER_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_FUNCTIONS_STMT,
   QUERY_NODE_SHOW_INDEXES_STMT,

include/libs/nodes/plannodes.h

@@ -155,7 +155,6 @@ typedef struct SLogicSubplan {
 
 typedef struct SQueryLogicPlan {
   ENodeType type;
-  int32_t totalLevel;
   SNodeList* pTopSubplans;
 } SQueryLogicPlan;
 

include/libs/nodes/querynodes.h

@@ -229,10 +229,10 @@ typedef struct SFillNode {
 typedef struct SSelectStmt {
   ENodeType type; // QUERY_NODE_SELECT_STMT
   bool isDistinct;
-  SNodeList* pProjectionList; // SNode
+  SNodeList* pProjectionList;
   SNode* pFromTable;
   SNode* pWhere;
-  SNodeList* pPartitionByList; // SNode
+  SNodeList* pPartitionByList;
   SNode* pWindow;
   SNodeList* pGroupByList; // SGroupingSetNode
   SNode* pHaving;
@@ -245,12 +245,14 @@ typedef struct SSelectStmt {
 } SSelectStmt;
 
 typedef enum ESetOperatorType {
-  SET_OP_TYPE_UNION_ALL = 1
+  SET_OP_TYPE_UNION_ALL = 1,
+  SET_OP_TYPE_UNION
 } ESetOperatorType;
 
 typedef struct SSetOperator {
   ENodeType type; // QUERY_NODE_SET_OPERATOR
   ESetOperatorType opType;
+  SNodeList* pProjectionList;
   SNode* pLeft;
   SNode* pRight;
   SNodeList* pOrderByList; // SOrderByExprNode
@@ -283,12 +285,12 @@ typedef struct SVgDataBlocks {
 } SVgDataBlocks;
 
 typedef struct SVnodeModifOpStmt {
-  ENodeType   nodeType;
+  ENodeType    nodeType;
-  ENodeType   sqlNodeType;
+  ENodeType    sqlNodeType;
-  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
+  SArray*      pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
-  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+  uint8_t      payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
-  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
+  uint32_t     insertType;          // insert data from [file|sql statement| bound statement]
-  const char* sql;                 // current sql statement position
+  const char*  sql;                 // current sql statement position
 } SVnodeModifOpStmt;
 
 typedef struct SExplainOptions {

include/libs/parser/parser.h

@@ -21,6 +21,15 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
+#include "query.h"
+
+typedef struct SStmtCallback {
+  TAOS_STMT* pStmt;
+  int32_t (*getTbNameFn)(TAOS_STMT*, char**);
+  int32_t (*setBindInfoFn)(TAOS_STMT*, STableMeta*, void*);
+  int32_t (*setExecInfoFn)(TAOS_STMT*, SHashObj*, SHashObj*);
+  int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
+} SStmtCallback;
 
 typedef struct SParseContext {
   uint64_t         requestId;
@@ -34,6 +43,7 @@ typedef struct SParseContext {
   char            *pMsg;           // extended error message if exists to help identifying the problem in sql statement.
   int32_t          msgLen;         // max length of the msg
   struct SCatalog *pCatalog;
+  SStmtCallback   *pStmtCb;
 } SParseContext;
 
 typedef struct SCmdMsgInfo {
@@ -66,11 +76,27 @@ typedef struct SQuery {
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
+bool isInsertSql(const char* pStr, size_t length);
 
 void qDestroyQuery(SQuery* pQueryNode);
 
 int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
 
+int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash);
+int32_t qResetStmtDataBlock(void* block, bool keepBuf);
+int32_t qCloneStmtDataBlock(void** pDst, void* pSrc);
+void    qFreeStmtDataBlock(void* pDataBlock);
+int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc);
+void    qDestroyStmtDataBlock(void* pBlock);
+int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum);
+int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
+int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
+int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+void destroyBoundColumnInfo(void* pBoundInfo);
+int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);
+
+
 #ifdef __cplusplus
 }
 #endif

include/libs/planner/planner.h

@@ -30,6 +30,7 @@ typedef struct SPlanContext {
   SNode* pAstRoot;
   bool topicQuery;
   bool streamQuery;
+  bool rSmaQuery;
   bool showRewrite;
   int8_t triggerType;
   int64_t watermark;
@@ -45,7 +46,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans 
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-typedef TAOS_MULTI_BIND TAOS_BIND_v2; // todo remove
 int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams);
 
 // Convert to subplan to string for the scheduler to send to the executor

include/libs/scalar/scalar.h

@@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 /* Time related functions */
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

include/util/talgo.h

@@ -27,6 +27,11 @@ extern "C" {
 typedef int32_t (*__compar_fn_t)(const void *, const void *);
 #endif
 
+typedef void *(*FCopy)(void *);
+typedef void (*FDelete)(void *);
+typedef int32_t (*FEncode)(void **buf, const void *dst);
+typedef void *(*FDecode)(const void *buf, void *dst);
+
 #define TD_EQ 0x1
 #define TD_GT 0x2
 #define TD_LT 0x4

include/util/taoserror.h

@@ -132,6 +132,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TSC_INVALID_JSON_TYPE         TAOS_DEF_ERROR_CODE(0, 0x0222)
 #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE        TAOS_DEF_ERROR_CODE(0, 0x0223)
 #define TSDB_CODE_TSC_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0X0224)
+#define TSDB_CODE_TSC_STMT_API_ERROR            TAOS_DEF_ERROR_CODE(0, 0X0225)
+#define TSDB_CODE_TSC_STMT_TBNAME_ERROR         TAOS_DEF_ERROR_CODE(0, 0X0226)
+#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR         TAOS_DEF_ERROR_CODE(0, 0X0227)
 
 // mnode-common
 #define TSDB_CODE_MND_APP_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x0300)
@@ -279,6 +282,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
@@ -609,6 +613,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INTER_SLIDING_UNIT        TAOS_DEF_ERROR_CODE(0, 0x2630)
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG     TAOS_DEF_ERROR_CODE(0, 0x2631)
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL   TAOS_DEF_ERROR_CODE(0, 0x2632)
+#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG         TAOS_DEF_ERROR_CODE(0, 0x2633)
+#define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL      TAOS_DEF_ERROR_CODE(0, 0x2634)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tarray.h

@@ -41,10 +41,10 @@ extern "C" {
 #define TARRAY_GET_START(array)       ((array)->pData)
 
 typedef struct SArray {
-  size_t size;
+  size_t   size;
   uint32_t capacity;
   uint32_t elemSize;
-  void*  pData;
+  void*    pData;
 } SArray;
 
 /**
@@ -199,6 +199,13 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize);
  */
 SArray* taosArrayDup(const SArray* pSrc);
 
+/**
+ * deep copy a new array
+ * @param pSrc
+ */
+SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
+
+
 /**
  * clear the array (remove all element)
  * @param pArray
@@ -212,19 +219,9 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
-
-/**
- * destroy array list
- * @param pArray
- */
 void* taosArrayDestroy(SArray* pArray);
-
+void  taosArrayDestroyP(SArray* pArray, FDelete fp);
-/**
+void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
- *
- * @param pArray
- * @param fp
- */
-void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
 
 /**
  * sort the array
@@ -272,6 +269,9 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t
 
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
 
+int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
+void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tcompare.h

@@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
 
+int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
+
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char *a, const char *b, int32_t type, size_t size);

include/util/tdef.h

@@ -239,6 +239,8 @@ typedef enum ELogicConditionType {
 #define TSDB_FUNC_BUF_SIZE       512
 #define TSDB_FUNC_TYPE_SCALAR    1
 #define TSDB_FUNC_TYPE_AGGREGATE 2
+#define TSDB_FUNC_SCRIPT_BIN_LIB 0
+#define TSDB_FUNC_SCRIPT_LUA     1
 #define TSDB_FUNC_MAX_RETRIEVE   1024
 
 #define TSDB_INDEX_NAME_LEN      65  // 64 + 1 '\0'
@@ -271,6 +273,8 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAGS           128
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
+#define TSDB_MAX_JSON_TAG_LEN   16384
+
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32
 #define TSDB_USET_PASSWORD_LEN 129

include/util/tjson.h

@@ -76,6 +76,7 @@ char* tjsonToString(const SJson* pJson);
 char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
+bool   tjsonValidateJson(const char* pJson);
 
 #ifdef __cplusplus
 }

include/util/tlog.h

@@ -59,6 +59,7 @@ extern int32_t sDebugFlag;
 extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
+extern int32_t fnDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();

include/util/tutil.h

@@ -26,8 +26,6 @@ extern "C" {
 #endif
 
 int32_t strdequote(char *src);
-int32_t strndequote(char *dst, const char *z, int32_t len);
-int32_t strRmquote(char *z, int32_t len);
 size_t  strtrim(char *src);
 char   *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
 char  **strsplit(char *src, const char *delim, int32_t *num);

packaging/install.sh

@@ -30,6 +30,15 @@ bin_dir="/usr/local/taos/bin"
 
 service_config_dir="/etc/systemd/system"
 
+#taos-tools para
+demoName="taosdemo"
+benchmarkName="taosBenchmark"
+dumpName="taosdump"
+emailName="taosdata.com"
+taosName="taos"
+toolsName="taostools"
+
+
 # Color setting
 RED='\033[0;31m'
 GREEN='\033[1;32m'
@@ -230,8 +239,20 @@ function install_header() {
 
 # temp install taosBenchmark
 function install_taosTools() {
-    cd ${script_dir}/taos-tools/
+    ${csudo} rm -f ${bin_link_dir}/${benchmarkName}    || :
-    tar xvf taosTools-1.4.1-Linux-x64.tar.gz && cd  taosTools-1.4.1/ && ./install-taostools.sh
+    ${csudo} rm -f ${bin_link_dir}/${dumpName}         || :
+    ${csudo} rm -f ${bin_link_dir}/rm${toolsName}      || :
+
+    ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName}
+    ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName}
+    ${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName}
+    #Make link
+    [[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \
+        ${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName}        || :
+    [[ -x ${install_main_dir}/bin/${demoName} ]] && \
+        ${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName}                  || :
+    [[ -x ${install_main_dir}/bin/${dumpName} ]] && \
+        ${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName}                  || :
 }
 
 function add_newHostname_to_hosts() {

packaging/release.sh

@@ -39,7 +39,7 @@ cd ${compile_dir}
 
 echo "compile_dir: ${compile_dir}"
 
-cmake ..
+cmake .. -DBUILD_TOOLS=true
 make -j32
 
 release_dir="${top_dir}/release"
@@ -55,7 +55,6 @@ mkdir -p ${install_dir}
 mkdir -p ${install_dir}/bin
 mkdir -p ${install_dir}/lib
 mkdir -p ${install_dir}/inc
-mkdir -p ${install_dir}/taos-tools
 
 install_files="${script_dir}/install.sh"
 chmod a+x ${script_dir}/install.sh || :
@@ -64,13 +63,14 @@ cp ${install_files} ${install_dir}
 header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h"
 cp ${header_files} ${install_dir}/inc
  
-bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos  ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/remove.sh"
+bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos  ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark   ${compile_dir}/build/bin/taosdump"
-cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
+cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
+
+cp -rf ${compile_dir}/source/client/libtaos.so  ${install_dir}/lib/
+cp -rf ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/
+cp -rf ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries"
+cp -rf ${compile_dir}/build/lib/pkgconfig  ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory"
 
-cp ${compile_dir}/build/lib/libtaos.so  ${install_dir}/lib/
-cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/
-taostoolfile="${top_dir}/tools/taosTools-1.4.1-Linux-x64.tar.gz"
-cp ${taostoolfile} ${install_dir}/taos-tools 
 
 #cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/
 #cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/

source/client/inc/clientInt.h

@@ -44,7 +44,7 @@ extern "C" {
   } while (0)
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
-#define HEARTBEAT_INTERVAL 1500  // ms
+#define HEARTBEAT_INTERVAL         1500  // ms
 
 enum {
   RES_TYPE__QUERY = 1,
@@ -187,11 +187,13 @@ typedef struct SRequestSendRecvBody {
 } SRequestSendRecvBody;
 
 typedef struct {
-  int8_t  resType;
+  int8_t         resType;
-  char*   topic;
+  char           topic[TSDB_TOPIC_FNAME_LEN];
-  SArray* res;  // SArray<SReqResultInfo>
+  int32_t        vgId;
-  int32_t resIter;
+  SSchemaWrapper schema;
-  int32_t vgId;
+  int32_t        resIter;
+  SMqDataBlkRsp  rsp;
+  SReqResultInfo resInfo;
 } SMqRspObj;
 
 typedef struct SRequestObj {
@@ -203,7 +205,8 @@ typedef struct SRequestObj {
   char*                sqlstr;  // sql string
   int32_t              sqlLen;
   int64_t              self;
-  char*                msgBuf;  // error msg buffer
+  char*                msgBuf;
+  int32_t              msgBufLen;
   int32_t              code;
   SArray*              dbList;
   SArray*              tableList;
@@ -211,16 +214,24 @@ typedef struct SRequestObj {
   SRequestSendRecvBody body;
 } SRequestObj;
 
+void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
+void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
+void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
+void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
+
 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  int32_t    resIter = msg->resIter == -1 ? 0 : msg->resIter;
+  return (SReqResultInfo*)&msg->resInfo;
-  return (SReqResultInfo*)taosArrayGet(msg->res, resIter);
 }
 
-static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) {
+static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  if (++msg->resIter < taosArrayGetSize(msg->res)) {
+  msg->resIter++;
-    return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter);
+  if (msg->resIter < msg->rsp.blockNum) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
+    setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4);
+    return &msg->resInfo;
   }
   return NULL;
 }
@@ -238,25 +249,25 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t
 int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
-int   taos_init();
+int taos_init();
 
-void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
+void*    createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
-void  destroyTscObj(void* pObj);
+void     destroyTscObj(void* pObj);
-STscObj *acquireTscObj(int64_t rid);
+STscObj* acquireTscObj(int64_t rid);
-int32_t releaseTscObj(int64_t rid);
+int32_t  releaseTscObj(int64_t rid);
 
 uint64_t generateRequestId();
 
-void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
+void*        createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
-void  destroyRequest(SRequestObj* pRequest);
+void         destroyRequest(SRequestObj* pRequest);
-SRequestObj *acquireRequest(int64_t rid);
+SRequestObj* acquireRequest(int64_t rid);
-int32_t releaseRequest(int64_t rid);
+int32_t      releaseRequest(int64_t rid);
 
 char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
 void  resetConnectDB(STscObj* pTscObj);
 
-int  taos_options_imp(TSDB_OPTION option, const char* str);
+int taos_options_imp(TSDB_OPTION option, const char* str);
 
 void* openTransporter(const char* user, const char* auth, int32_t numOfThreads);
 
@@ -268,17 +279,12 @@ void initMsgHandleFp();
 TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                             uint16_t port, int connType);
 
-int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
+int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);
+
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
-void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
-void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
-void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
-void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
-int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
-
 // --- heartbeat
 // global, called by mgmt
 int  hbMgrInit();
@@ -290,7 +296,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
 void       appHbMgrCleanup(void);
 
 // conn level
-int  hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
+int  hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
 void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
 
 int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen);
@@ -298,6 +304,8 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
 // --- mq
 void hbMgrInitMqHbRspHandle();
 
+SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
+
 #ifdef __cplusplus
 }
 #endif

source/client/inc/clientStmt.h

@@ -19,6 +19,9 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "catalog.h"
+
+typedef void STableDataBlocks;
 
 typedef enum {
   STMT_TYPE_INSERT = 1,
@@ -26,17 +29,64 @@ typedef enum {
   STMT_TYPE_QUERY,
 } STMT_TYPE;
 
-typedef struct STscStmt {
+typedef enum {
-  STMT_TYPE type;
+  STMT_INIT = 1,
-  //int16_t  last;
+  STMT_PREPARE,
-  //STscObj* taos;
+  STMT_SETTBNAME,
-  //SSqlObj* pSql;
+  STMT_SETTAGS,
-  //SMultiTbStmt mtb;
+  STMT_FETCH_TAG_FIELDS,
-  //SNormalStmt normal;
+  STMT_FETCH_COL_FIELDS,
+  STMT_BIND,
+  STMT_BIND_COL,
+  STMT_ADD_BATCH,
+  STMT_EXECUTE,
+} STMT_STATUS;
 
-  //int numOfRows;
+typedef struct SStmtTableCache {
+  STableDataBlocks* pDataBlock;
+  void*             boundTags;
+} SStmtTableCache;
+
+typedef struct SStmtBindInfo {
+  bool         needParse;
+  uint64_t     tbUid;
+  uint64_t     tbSuid;
+  int32_t      sBindRowNum;
+  int32_t      sBindLastIdx;
+  int8_t       tbType;
+  void*        boundTags;  
+  char*        tbName;
+  SName        sname;
+} SStmtBindInfo;
+
+typedef struct SStmtExecInfo {
+  SRequestObj* pRequest;
+  SHashObj*    pVgHash;
+  SHashObj*    pBlockHash;
+} SStmtExecInfo;
+
+typedef struct SStmtSQLInfo {
+  STMT_TYPE    type;
+  STMT_STATUS  status;
+  bool         autoCreate;
+  uint64_t     runTimes;
+  SHashObj*    pTableCache;   //SHash<SStmtTableCache>
+  SQuery*      pQuery;
+  char*        sqlStr;
+  int32_t      sqlLen;
+} SStmtSQLInfo;
+
+typedef struct STscStmt {
+  STscObj*      taos;
+  SCatalog*     pCatalog;
+  int32_t       affectedRows;
+
+  SStmtSQLInfo  sql;
+  SStmtExecInfo exec;
+  SStmtBindInfo bInfo;
 } STscStmt;
 
+
 #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
 #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
 #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
@@ -44,16 +94,16 @@ typedef struct STscStmt {
 TAOS_STMT *stmtInit(TAOS *taos);
 int stmtClose(TAOS_STMT *stmt);
 int stmtExec(TAOS_STMT *stmt);
-char *stmtErrstr(TAOS_STMT *stmt);
+const char *stmtErrstr(TAOS_STMT *stmt);
 int stmtAffectedRows(TAOS_STMT *stmt);
-int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind);
 int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags);
+int stmtSetTbName(TAOS_STMT *stmt, const char *tbName);
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags);
 int stmtIsInsert(TAOS_STMT *stmt, int *insert);
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums);
 int stmtAddBatch(TAOS_STMT *stmt);
 TAOS_RES *stmtUseResult(TAOS_STMT *stmt);
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx);
 
 
 #ifdef __cplusplus

source/client/src/clientEnv.c

@@ -186,6 +186,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
   pRequest->pTscObj = pObj;
   pRequest->body.fp = fp;  // not used it yet
   pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
+  pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE;
   tsem_init(&pRequest->body.rspSem, 0, 0);
 
   registerRequest(pRequest);

source/client/src/clientImpl.c

@@ -146,7 +146,8 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
   (*pRequest)->sqlstr[sqlLen] = 0;
   (*pRequest)->sqlLen = sqlLen;
 
-  if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) {
+  if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
+                  sizeof((*pRequest)->self))) {
     destroyRequest(*pRequest);
     *pRequest = NULL;
     tscError("put request to request hash failed");
@@ -157,7 +158,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
+int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb) {
   STscObj* pTscObj = pRequest->pTscObj;
 
   SParseContext cxt = {
@@ -170,6 +171,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
       .pMsg = pRequest->msgBuf,
       .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
       .pTransporter = pTscObj->pAppInfo->pTransporter,
+      .pStmtCb = pStmtCb,
   };
 
   cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
@@ -262,7 +264,8 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
 }
 
 void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) {
-  if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO && precision != TSDB_TIME_PRECISION_NANO) {
+  if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO &&
+      precision != TSDB_TIME_PRECISION_NANO) {
     return;
   }
 
@@ -274,7 +277,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
 
   SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
   int32_t      code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr,
-                                       pRequest->metric.start, &res);
+                                  pRequest->metric.start, &res);
   if (code != TSDB_CODE_SUCCESS) {
     if (pRequest->body.queryJob != 0) {
       schedulerFreeJob(pRequest->body.queryJob);
@@ -298,15 +301,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
   return pRequest->code;
 }
 
-SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
+SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
-  SRequestObj* pRequest = NULL;
+  SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
-  SQuery*      pQuery = NULL;
-  SArray*      pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
-
-  int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = parseSql(pRequest, false, &pQuery);
-  }
 
   if (TSDB_CODE_SUCCESS == code) {
     switch (pQuery->execMode) {
@@ -331,7 +327,10 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
   }
 
   taosArrayDestroy(pNodeList);
-  qDestroyQuery(pQuery);
+  if (!keepQuery) {
+    qDestroyQuery(pQuery);
+  }
+
   if (NULL != pRequest && TSDB_CODE_SUCCESS != code) {
     pRequest->code = terrno;
   }
@@ -339,6 +338,18 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
   return pRequest;
 }
 
+SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
+  SRequestObj* pRequest = NULL;
+  SQuery*      pQuery = NULL;
+
+  int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = parseSql(pRequest, false, &pQuery, NULL);
+  }
+
+  return launchQueryImpl(pRequest, pQuery, code, false);
+}
+
 int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
   SCatalog* pCatalog = NULL;
   int32_t   code = 0;
@@ -383,7 +394,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   int32_t      code = 0;
 
   while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
-    pRequest = execQueryImpl(pTscObj, sql, sqlLen);
+    pRequest = launchQuery(pTscObj, sql, sqlLen);
     if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
       break;
     }
@@ -512,6 +523,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) {
   connectReq.pid = htonl(appInfo.pid);
   connectReq.startTime = htobe64(appInfo.startTime);
   tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app));
+  tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user));
+  tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd));
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = taosMemoryMalloc(contLen);
@@ -715,6 +728,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
 
           int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p));
           ASSERT(len <= bytes);
+          ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i]));
 
           varDataSetLen(p, len);
           pCol->offset[j] = (p - pResultInfo->convertBuf[i]);
@@ -725,6 +739,80 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
       pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
       pResultInfo->row[i] = pResultInfo->pCol[i].pData;
     }
+
+    if (type == TSDB_DATA_TYPE_JSON) {
+      char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
+      if (p == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+
+      pResultInfo->convertBuf[i] = p;
+      int32_t        len = 0;
+      SResultColumn* pCol = &pResultInfo->pCol[i];
+      for (int32_t j = 0; j < numOfRows; ++j) {
+        if (pCol->offset[j] != -1) {
+          char* pStart = pCol->offset[j] + pCol->pData;
+
+          int32_t jsonInnerType = *pStart;
+          char*   jsonInnerData = pStart + CHAR_BYTES;
+          char    dst[TSDB_MAX_JSON_TAG_LEN] = {0};
+          if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
+            sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) {
+            int32_t length =
+                taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
+
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+                       varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length);
+          } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {  // value -> "value"
+            *(char*)varDataVal(dst) = '\"';
+            int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
+                                           varDataVal(dst) + CHAR_BYTES);
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+                       varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length + CHAR_BYTES * 2);
+            *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
+          } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
+            double jsonVd = *(double*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%.9lf", jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) {
+            int64_t jsonVd = *(int64_t*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%" PRId64, jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
+            sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else {
+            ASSERT(0);
+          }
+
+          if (len + varDataTLen(dst) > colLength[i]) {
+            p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
+            if (p == NULL) {
+              return TSDB_CODE_OUT_OF_MEMORY;
+            }
+
+            pResultInfo->convertBuf[i] = p;
+          }
+          p = pResultInfo->convertBuf[i] + len;
+          memcpy(p, dst, varDataTLen(dst));
+          pCol->offset[j] = len;
+          len += varDataTLen(dst);
+        }
+      }
+
+      pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
+      pResultInfo->row[i] = pResultInfo->pCol[i].pData;
+    }
   }
 
   return TSDB_CODE_SUCCESS;

source/client/src/clientMain.c

@@ -14,12 +14,12 @@
  */
 
 #include "catalog.h"
-#include "scheduler.h"
 #include "clientInt.h"
-#include "clientStmt.h"
 #include "clientLog.h"
+#include "clientStmt.h"
 #include "os.h"
 #include "query.h"
+#include "scheduler.h"
 #include "tglobal.h"
 #include "tmsg.h"
 #include "tref.h"
@@ -128,6 +128,10 @@ const char *taos_errstr(TAOS_RES *res) {
 }
 
 void taos_free_result(TAOS_RES *res) {
+  if (NULL == res) {
+    return;
+  }
+  
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
     destroyRequest(pRequest);
@@ -177,25 +181,24 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return doFetchRows(pRequest, true, true);
 
   } else if (TD_RES_TMQ(res)) {
-    SMqRspObj *msg = ((SMqRspObj *)res);
+    SMqRspObj      *msg = ((SMqRspObj *)res);
-    if (msg->resIter == -1) msg->resIter++;
+    SReqResultInfo *pResultInfo;
-    SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter);
+    if (msg->resIter == -1) {
+      pResultInfo = tmqGetNextResInfo(res, true);
+    } else {
+      pResultInfo = tmqGetCurResInfo(res);
+    }
     if (pResultInfo->current < pResultInfo->numOfRows) {
       doSetOneRowPtr(pResultInfo);
       pResultInfo->current += 1;
       return pResultInfo->row;
     } else {
-      msg->resIter++;
+      pResultInfo = tmqGetNextResInfo(res, true);
-      if (msg->resIter < taosArrayGetSize(msg->res)) {
+      if (pResultInfo == NULL) return NULL;
-        pResultInfo = taosArrayGet(msg->res, msg->resIter);
+      doSetOneRowPtr(pResultInfo);
-        doSetOneRowPtr(pResultInfo);
+      pResultInfo->current += 1;
-        pResultInfo->current += 1;
+      return pResultInfo->row;
-        return pResultInfo->row;
-      } else {
-        return NULL;
-      }
     }
-
   } else {
     // assert to avoid un-initialization error
     ASSERT(0);
@@ -455,7 +458,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
     (*numOfRows) = pResultInfo->numOfRows;
     return pRequest->code;
   } else if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true);
     if (pResultInfo == NULL) return -1;
 
     pResultInfo->current = pResultInfo->numOfRows;
@@ -474,7 +477,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
   }
 
   if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false);
     if (pResultInfo == NULL) {
       (*numOfRows) = 0;
       return 0;
@@ -580,56 +583,6 @@ TAOS_STMT *taos_stmt_init(TAOS *taos) {
   return stmtInit(taos);
 }
 
-int taos_stmt_close(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
-    tscError("NULL parameter for %s", __FUNCTION__);
-    terrno = TSDB_CODE_INVALID_PARA;
-    return terrno;
-  }
-
-  return stmtClose(stmt);
-}
-
-int taos_stmt_execute(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
-    tscError("NULL parameter for %s", __FUNCTION__);
-    terrno = TSDB_CODE_INVALID_PARA;
-    return terrno;
-  }
-
-  return stmtExec(stmt);
-}
-
-char *taos_stmt_errstr(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
-    tscError("NULL parameter for %s", __FUNCTION__);
-    terrno = TSDB_CODE_INVALID_PARA;
-    return NULL;
-  }
-
-  return stmtErrstr(stmt);
-}
-
-int taos_stmt_affected_rows(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
-    tscError("NULL parameter for %s", __FUNCTION__);
-    terrno = TSDB_CODE_INVALID_PARA;
-    return 0;
-  }
-
-  return stmtAffectedRows(stmt);
-}
-
-int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) {
-  if (stmt == NULL || bind == NULL) {
-    tscError("NULL parameter for %s", __FUNCTION__);
-    terrno = TSDB_CODE_INVALID_PARA;
-    return terrno;
-  }
-
-  return stmtBind(stmt, bind);
-}
-
 int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
   if (stmt == NULL || sql == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
@@ -640,14 +593,23 @@ int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
   return stmtPrepare(stmt, sql, length);
 }
 
-int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) {
+int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) {
-  if (stmt == NULL || name == NULL || tags == NULL) {
+  if (stmt == NULL || name == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtSetTbNameTags(stmt, name, tags);
+  int32_t code = stmtSetTbName(stmt, name);
+  if (code) {
+    return code;
+  }
+
+  if (tags) {
+    return stmtSetTbTags(stmt, tags);
+  }
+
+  return TSDB_CODE_SUCCESS;
 }
 
 int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
@@ -657,7 +619,75 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
     return terrno;
   }
 
-  return stmtSetTbNameTags(stmt, name, NULL);
+  return stmtSetTbName(stmt, name);
+}
+
+int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
+  if (stmt == NULL || bind == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  if (bind->num > 1) {
+    tscError("invalid bind number %d for %s", bind->num, __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  
+  return stmtBindBatch(stmt, bind, -1);
+}
+
+int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
+  if (stmt == NULL || bind == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  if (bind->num <= 0 || bind->num > INT16_MAX) {
+    tscError("invalid bind num %d", bind->num);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  return stmtBindBatch(stmt, bind, -1);
+}
+
+int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) {
+  if (stmt == NULL || bind == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  if (colIdx < 0) {
+    tscError("invalid bind column idx %d", colIdx);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  
+  return stmtBindBatch(stmt, bind, colIdx);
+}
+
+int taos_stmt_add_batch(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  return stmtAddBatch(stmt);
+}
+
+int taos_stmt_execute(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  return stmtExec(stmt);
 }
 
 int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
@@ -680,16 +710,6 @@ int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
   return stmtGetParamNum(stmt, nums);
 }
 
-int taos_stmt_add_batch(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
-    tscError("NULL parameter for %s", __FUNCTION__);
-    terrno = TSDB_CODE_INVALID_PARA;
-    return terrno;
-  }
-
-  return stmtAddBatch(stmt);
-}
-
 TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
   if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
@@ -700,20 +720,32 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
   return stmtUseResult(stmt);
 }
 
-int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
+char *taos_stmt_errstr(TAOS_STMT *stmt) {
-  if (stmt == NULL || bind == NULL) {
+  return (char *)stmtErrstr(stmt);
+}
+
+int taos_stmt_affected_rows(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return 0;
+  }
+
+  return stmtAffectedRows(stmt);
+}
+
+int taos_stmt_close(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtBindBatch(stmt, bind);
+  return stmtClose(stmt);
 }
 
-
 TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
   // TODO
   return NULL;
 }
 
-

source/client/src/clientStmt.c

@@ -4,86 +4,539 @@
 #include "clientStmt.h"
 #include "tdef.h"
 
+int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
+  switch (newStatus) {
+    case STMT_SETTBNAME:
+      break;
+    default:
+      break;
+  }
+
+  //STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+
+  pStmt->sql.status = newStatus;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  pStmt->sql.type = STMT_TYPE_MULTI_INSERT;
+  
+  if (NULL == pStmt->bInfo.tbName) {
+    tscError("no table name set");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR);
+  }
+
+  *tbName = pStmt->bInfo.tbName;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  pStmt->bInfo.tbUid = pTableMeta->uid;
+  pStmt->bInfo.tbSuid = pTableMeta->suid;
+  pStmt->bInfo.tbType = pTableMeta->tableType;
+  pStmt->bInfo.boundTags = tags;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtSetExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  pStmt->exec.pVgHash = pVgHash;
+  pStmt->exec.pBlockHash = pBlockHash;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  *pVgHash = pStmt->exec.pVgHash;
+  *pBlockHash = pStmt->exec.pBlockHash;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCacheBlock(STscStmt *pStmt) {
+  if (pStmt->sql.type != STMT_TYPE_MULTI_INSERT) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  uint64_t uid;
+  if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) {
+    uid = pStmt->bInfo.tbSuid;
+  } else {
+    ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType);
+    uid = pStmt->bInfo.tbUid;
+  }
+
+  if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid));
+  STableDataBlocks* pDst = NULL;
+  
+  STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc));
+
+  SStmtTableCache cache = {
+    .pDataBlock = pDst,
+    .boundTags = pStmt->bInfo.boundTags,
+  };
+
+  if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pStmt->bInfo.boundTags = NULL;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtParseSql(STscStmt* pStmt) {
+  SStmtCallback stmtCb = {
+    .pStmt = pStmt, 
+    .getTbNameFn = stmtGetTbName, 
+    .setBindInfoFn = stmtSetBindInfo,
+    .setExecInfoFn = stmtSetExecInfo,
+    .getExecInfoFn = stmtGetExecInfo,
+  };
+
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+  
+  STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb));
+
+  pStmt->bInfo.needParse = false;
+  
+  switch (nodeType(pStmt->sql.pQuery->pRoot)) {
+    case QUERY_NODE_VNODE_MODIF_STMT:
+      if (0 == pStmt->sql.type) {
+        pStmt->sql.type = STMT_TYPE_INSERT;
+      }
+      break;
+    case QUERY_NODE_SELECT_STMT:
+      pStmt->sql.type = STMT_TYPE_QUERY;
+      break;
+    default:
+      tscError("not supported stmt type %d", nodeType(pStmt->sql.pQuery->pRoot));
+      STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR);
+  }
+
+  STMT_ERR_RET(stmtCacheBlock(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCleanBindInfo(STscStmt* pStmt) {
+  pStmt->bInfo.tbUid = 0;
+  pStmt->bInfo.tbSuid = 0;
+  pStmt->bInfo.tbType = 0;
+  pStmt->bInfo.needParse = true;
+
+  taosMemoryFreeClear(pStmt->bInfo.tbName);
+  destroyBoundColumnInfo(pStmt->bInfo.boundTags);
+  taosMemoryFreeClear(pStmt->bInfo.boundTags);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) {
+  taos_free_result(pStmt->exec.pRequest);
+  pStmt->exec.pRequest = NULL;
+
+  void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
+  while (pIter) {
+    STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter;    
+    uint64_t *key = taosHashGetKey(pIter, NULL);
+    
+    if (keepTable && (*key == pStmt->bInfo.tbUid)) {
+      STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true));
+      
+      pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter);
+      continue;
+    }
+
+    qFreeStmtDataBlock(pBlocks);
+    taosHashRemove(pStmt->exec.pBlockHash, key, sizeof(*key));
+
+    pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter);
+  }
+
+  if (keepTable) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  taosHashCleanup(pStmt->exec.pBlockHash);
+  pStmt->exec.pBlockHash = NULL;
+
+  STMT_ERR_RET(stmtCleanBindInfo(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
+  taosMemoryFree(pStmt->sql.sqlStr);
+  qDestroyQuery(pStmt->sql.pQuery);
+
+  void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL);
+  while (pIter) {
+    SStmtTableCache* pCache = (SStmtTableCache*)pIter;    
+
+    qDestroyStmtDataBlock(pCache->pDataBlock);
+    destroyBoundColumnInfo(pCache->boundTags);
+    
+    pIter = taosHashIterate(pStmt->sql.pTableCache, pIter);
+  }
+  taosHashCleanup(pStmt->sql.pTableCache);
+  pStmt->sql.pTableCache = NULL;
+
+  memset(&pStmt->sql, 0, sizeof(pStmt->sql));
+
+  STMT_ERR_RET(stmtCleanExecInfo(pStmt, false));
+  STMT_ERR_RET(stmtCleanBindInfo(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtGetFromCache(STscStmt* pStmt) {
+  pStmt->bInfo.needParse = true;
+
+  if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (NULL == pStmt->pCatalog) {
+    STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog));
+  }
+
+  STableMeta *pTableMeta = NULL;
+  SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
+  STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
+
+  if (pTableMeta->uid == pStmt->bInfo.tbUid) {
+    pStmt->bInfo.needParse = false;
+    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) {
+    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+    if (NULL == pCache) {
+      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid);
+      STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
+    }
+    
+    pStmt->bInfo.needParse = false;
+    
+    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.boundTags = pCache->boundTags;
+    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+  if (pCache) {
+    pStmt->bInfo.needParse = false;
+
+    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.boundTags = pCache->boundTags;
+
+    STableDataBlocks* pNewBlock = NULL;
+    STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock));
+
+    if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid), &pNewBlock, POINTER_BYTES)) {
+      STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  STMT_ERR_RET(stmtCleanBindInfo(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtResetStmt(STscStmt* pStmt) {
+  STMT_ERR_RET(stmtCleanSQLInfo(pStmt));
+
+  pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  if (NULL == pStmt->sql.pTableCache) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    STMT_ERR_RET(terrno);
+  }
+
+  pStmt->sql.status = STMT_INIT;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 TAOS_STMT *stmtInit(TAOS *taos) {
   STscObj* pObj = (STscObj*)taos;
   STscStmt* pStmt = NULL;
 
-#if 0
   pStmt = taosMemoryCalloc(1, sizeof(STscStmt));
-  if (pStmt == NULL) {
+  if (NULL == pStmt) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscError("failed to allocate memory for statement");
     return NULL;
   }
+
+  pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  if (NULL == pStmt->sql.pTableCache) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    taosMemoryFree(pStmt);
+    return NULL;
+  }
+
   pStmt->taos = pObj;
-
+  pStmt->bInfo.needParse = true;
-  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  pStmt->sql.status = STMT_INIT;
-
+  
-  if (pSql == NULL) {
-    free(pStmt);
-    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscError("failed to allocate memory for statement");
-    return NULL;
-  }
-
-  if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
-    free(pSql);
-    free(pStmt);
-    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscError("failed to malloc payload buffer");
-    return NULL;
-  }
-
-  tsem_init(&pSql->rspSem, 0, 0);
-  pSql->signature   = pSql;
-  pSql->pTscObj     = pObj;
-  pSql->maxRetry    = TSDB_MAX_REPLICA;
-  pStmt->pSql       = pSql;
-  pStmt->last       = STMT_INIT;
-  pStmt->numOfRows  = 0;
-  registerSqlObj(pSql);
-#endif
-
   return pStmt;
 }
 
-int stmtClose(TAOS_STMT *stmt) {
+int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (pStmt->sql.status >= STMT_PREPARE) {
+    STMT_ERR_RET(stmtResetStmt(pStmt));
+  }
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_PREPARE));
+
+  if (length <= 0) {
+    length = strlen(sql);
+  }
+  
+  pStmt->sql.sqlStr = strndup(sql, length);
+  pStmt->sql.sqlLen = length;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
+
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+  
+  STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
+    
+  STMT_ERR_RET(stmtGetFromCache(pStmt));
+
+  if (pStmt->bInfo.needParse) {
+    taosMemoryFree(pStmt->bInfo.tbName);
+    pStmt->bInfo.tbName = strdup(tbName);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS));
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    tscError("invalid operation to get query tag fileds");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS));
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    tscError("invalid operation to get query column fileds");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields));
+
+  return TSDB_CODE_SUCCESS;  
+}
+
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
+
+  if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
+    pStmt->bInfo.needParse = false;
+  }
+
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  if (colIdx < 0) {
+    qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
+  } else {
+    if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) {
+      tscError("bind column index not in sequence");
+      STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+    }
+
+    pStmt->bInfo.sBindLastIdx = colIdx;
+    
+    if (0 == colIdx) {
+      pStmt->bInfo.sBindRowNum = bind->num;
+    }
+    
+    qBindStmtSingleColValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum);
+  }
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int stmtAddBatch(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH));
+
+  STMT_ERR_RET(stmtCacheBlock(pStmt));
+  
   return TSDB_CODE_SUCCESS;
 }
 
 int stmtExec(TAOS_STMT *stmt) {
-  return TSDB_CODE_SUCCESS;
+  STscStmt* pStmt = (STscStmt*)stmt;
+  int32_t code = 0;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
+
+  STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
+
+  launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
+
+  STMT_ERR_JRET(pStmt->exec.pRequest->code);
+
+  pStmt->affectedRows += taos_affected_rows(pStmt->exec.pRequest);
+
+_return:
+
+  stmtCleanExecInfo(pStmt, (code ? false : true));
+  
+  ++pStmt->sql.runTimes;
+  
+  STMT_RET(code);
 }
 
-char *stmtErrstr(TAOS_STMT *stmt) {
+
-  return NULL;
+int stmtClose(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_RET(stmtCleanSQLInfo(pStmt));
+}
+
+const char *stmtErrstr(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (stmt == NULL) {
+    return (char*) tstrerror(terrno);
+  }
+
+  if (pStmt->exec.pRequest) {
+    pStmt->exec.pRequest->code = terrno;
+  }
+
+  return taos_errstr(pStmt->exec.pRequest);
 }
 
 int stmtAffectedRows(TAOS_STMT *stmt) {
-  return TSDB_CODE_SUCCESS;
+  return ((STscStmt*)stmt)->affectedRows;
-}
-
-int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind) {
-  return TSDB_CODE_SUCCESS;
-}
-
-int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
-  return TSDB_CODE_SUCCESS;
-}
-
-int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) {
-  return TSDB_CODE_SUCCESS;
 }
 
 int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (pStmt->sql.type) {
+    *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type);
+  } else {
+    *insert = isInsertSql(pStmt->sql.sqlStr, 0);
+  }
+  
   return TSDB_CODE_SUCCESS;
 }
 
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
-  return TSDB_CODE_SUCCESS;
+  STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
-}
+  
-
-int stmtAddBatch(TAOS_STMT *stmt) {
   return TSDB_CODE_SUCCESS;
 }
 
@@ -91,9 +544,5 @@ TAOS_RES *stmtUseResult(TAOS_STMT *stmt) {
   return NULL;
 }
 
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
-  return TSDB_CODE_SUCCESS;
-}
-
 
 

source/client/src/tmq.c

@@ -24,6 +24,7 @@
 #include "tqueue.h"
 #include "tref.h"
 
+#if 0
 struct tmq_message_t {
   SMqPollRsp msg;
   char*      topic;
@@ -31,6 +32,7 @@ struct tmq_message_t {
   int32_t    vgId;
   int32_t    resIter;
 };
+#endif
 
 typedef struct {
   int8_t  tmqRspType;
@@ -52,9 +54,7 @@ struct tmq_topic_vgroup_t {
 };
 
 struct tmq_topic_vgroup_list_t {
-  int32_t             cnt;
+  SArray container;  // SArray<tmq_topic_vgroup_t*>
-  int32_t             size;
-  tmq_topic_vgroup_t* elems;
 };
 
 struct tmq_conf_t {
@@ -63,6 +63,7 @@ struct tmq_conf_t {
   int8_t         autoCommit;
   int8_t         resetOffset;
   uint16_t       port;
+  uint16_t       autoCommitInterval;
   char*          ip;
   char*          user;
   char*          pass;
@@ -72,25 +73,25 @@ struct tmq_conf_t {
 
 struct tmq_t {
   // conf
-  char           groupId[TSDB_CGROUP_LEN];
+  char   groupId[TSDB_CGROUP_LEN];
-  char           clientId[256];
+  char   clientId[256];
-  int8_t         autoCommit;
+  int8_t autoCommit;
-  int8_t         inWaiting;
+  /*int8_t         inWaiting;*/
   int64_t        consumerId;
   int32_t        epoch;
   int32_t        resetOffsetCfg;
   int64_t        status;
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
-  int32_t        nextTopicIdx;
+  /*int32_t        nextTopicIdx;*/
-  int8_t         epStatus;
+  int8_t  epStatus;
-  int32_t        epSkipCnt;
+  int32_t epSkipCnt;
-  int32_t        waitingRequest;
+  /*int32_t        waitingRequest;*/
-  int32_t        readyRequest;
+  /*int32_t        readyRequest;*/
-  SArray*        clientTopics;  // SArray<SMqClientTopic>
+  SArray*     clientTopics;  // SArray<SMqClientTopic>
-  STaosQueue*    mqueue;        // queue of tmq_message_t
+  STaosQueue* mqueue;        // queue of tmq_message_t
-  STaosQall*     qall;
+  STaosQall*  qall;
-  tsem_t         rspSem;
+  tsem_t      rspSem;
   // stat
   int64_t pollCnt;
 };
@@ -134,7 +135,7 @@ typedef struct {
   int32_t         epoch;
   SMqClientVg*    vgHandle;
   SMqClientTopic* topicHandle;
-  SMqPollRspV2    msg;
+  SMqDataBlkRsp   msg;
 } SMqPollRspWrapper;
 
 typedef struct {
@@ -145,6 +146,7 @@ typedef struct {
 
 typedef struct {
   tmq_t*  tmq;
+  int32_t code;
   int32_t sync;
   tsem_t  rspSem;
 } SMqAskEpCbParam;
@@ -201,6 +203,11 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     }
   }
 
+  if (strcmp(key, "auto.commit.interval.ms") == 0) {
+    conf->autoCommitInterval = atoi(value);
+    return TMQ_CONF_OK;
+  }
+
   if (strcmp(key, "auto.offset.reset") == 0) {
     if (strcmp(value, "none") == 0) {
       conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE;
@@ -255,7 +262,12 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
 void tmq_list_destroy(tmq_list_t* list) {
   SArray* container = &list->container;
   /*taosArrayDestroy(container);*/
-  taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree);
+  int32_t sz = taosArrayGetSize(container);
+  for (int32_t i = 0; i < sz; i++) {
+    char* str = taosArrayGetP(container, i);
+    taosMemoryFree(str);
+  }
+  taosArrayDestroy(container);
 }
 
 static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
@@ -294,7 +306,7 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
   pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
   if (pParam->tmq->commit_cb) {
-    pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL);
+    pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL);
   }
   if (!pParam->async) tsem_post(&pParam->rspSem);
   return 0;
@@ -316,18 +328,19 @@ tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) {
   return tmq_subscribe(tmq, lst);
 }
 
+#if 0
 tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   tmq_t* pTmq = taosMemoryCalloc(sizeof(tmq_t), 1);
   if (pTmq == NULL) {
     return NULL;
   }
   pTmq->pTscObj = (STscObj*)conn;
-  pTmq->inWaiting = 0;
+  /*pTmq->inWaiting = 0;*/
   pTmq->status = 0;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
-  pTmq->waitingRequest = 0;
+  /*pTmq->waitingRequest = 0;*/
-  pTmq->readyRequest = 0;
+  /*pTmq->readyRequest = 0;*/
   pTmq->epStatus = 0;
   pTmq->epSkipCnt = 0;
   // set conf
@@ -351,8 +364,9 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
 
   return pTmq;
 }
+#endif
 
-tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
+tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t));
   if (pTmq == NULL) {
     return NULL;
@@ -363,16 +377,17 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   ASSERT(user);
   ASSERT(pass);
   ASSERT(conf->db);
+  ASSERT(conf->groupId[0]);
 
   pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ);
   if (pTmq->pTscObj == NULL) return NULL;
 
-  pTmq->inWaiting = 0;
+  /*pTmq->inWaiting = 0;*/
   pTmq->status = 0;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
-  pTmq->waitingRequest = 0;
+  /*pTmq->waitingRequest = 0;*/
-  pTmq->readyRequest = 0;
+  /*pTmq->readyRequest = 0;*/
   pTmq->epStatus = 0;
   pTmq->epSkipCnt = 0;
   // set conf
@@ -423,8 +438,8 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
     req.num = pArray->size;
     req.offsets = pArray->pData;
   } else {
-    req.num = offsets->cnt;
+    req.num = taosArrayGetSize(&offsets->container);
-    req.offsets = (SMqOffset*)offsets->elems;
+    req.offsets = (SMqOffset*)offsets->container.pData;
   }
 
   SCoder encoder;
@@ -496,7 +511,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
   SCMSubscribeReq req;
   req.topicNum = sz;
   req.consumerId = tmq->consumerId;
-  req.consumerGroup = strdup(tmq->groupId);
+  strcpy(req.cgroup, tmq->groupId);
   req.topicNames = taosArrayInit(sz, sizeof(void*));
 
   for (int i = 0; i < sz; i++) {
@@ -609,7 +624,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
 
   int32_t code = 0;
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
-  CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return);
+  CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return);
 
   // todo check for invalid sql statement and return with error code
 
@@ -857,7 +872,6 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
 #endif
 
 int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
-  /*printf("recv poll\n");*/
   SMqPollCbParam* pParam = (SMqPollCbParam*)param;
   SMqClientVg*    pVg = pParam->pVg;
   SMqClientTopic* pTopic = pParam->pTopic;
@@ -870,17 +884,15 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
-    /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
+    // do not write into queue since updating epoch reset
-    /*tsem_post(&tmq->rspSem);*/
     tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
             tmqEpoch);
+    /*tsem_post(&tmq->rspSem);*/
     return 0;
   }
 
   if (msgEpoch != tmqEpoch) {
     tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
-  } else {
-    atomic_sub_fetch_32(&tmq->waitingRequest, 1);
   }
 
 #if 0
@@ -902,45 +914,33 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 #endif
 
-  /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
-  /*tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));*/
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper));
   if (pRspWrapper == NULL) {
     tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
     goto CREATE_MSG_FAIL;
   }
+
   pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP;
   pRspWrapper->vgHandle = pVg;
   pRspWrapper->topicHandle = pTopic;
-  /*memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));*/
-  memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
-  tDecodeSMqPollRspV2(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
-  // TODO: alloc mem
-  /*pRsp->*/
-  /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
 
-#if 0
+  memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
-  if (pRsp->msg.numOfTopics == 0) {
+
-    /*printf("no data\n");*/
+  tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
-    taosFreeQitem(pRsp);
-    goto CREATE_MSG_FAIL;
-  }
-#endif
 
   tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId,
            pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
-  atomic_add_fetch_32(&tmq->readyRequest, 1);
   /*tsem_post(&tmq->rspSem);*/
-  return 0;
 
+  return 0;
 CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
   /*tsem_post(&tmq->rspSem);*/
-  return code;
+  return -1;
 }
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
@@ -1023,6 +1023,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
 int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
   tmq_t*           tmq = pParam->tmq;
+  pParam->code = code;
   if (code != 0) {
     tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync);
     goto END;
@@ -1062,6 +1063,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
 
     taosWriteQitem(tmq->mqueue, pWrapper);
     /*tsem_post(&tmq->rspSem);*/
+    taosMemoryFree(pParam);
   }
 
 END:
@@ -1073,7 +1075,8 @@ END:
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
-  int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
+  int32_t code = 0;
+  int8_t  epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
   if (epStatus == 1) {
     int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
     tscTrace("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
@@ -1130,8 +1133,12 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
-  if (sync) tsem_wait(&pParam->rspSem);
+  if (sync) {
-  return 0;
+    tsem_wait(&pParam->rspSem);
+    code = pParam->code;
+    taosMemoryFree(pParam);
+  }
+  return code;
 }
 
 tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
@@ -1157,7 +1164,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
   return TMQ_RESP_ERR__FAIL;
 }
 
-SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
   int64_t reqOffset;
   if (pVg->currentOffset >= 0) {
     reqOffset = pVg->currentOffset;
@@ -1169,13 +1176,18 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
     reqOffset = tmq->resetOffsetCfg;
   }
 
-  SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq));
+  SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2));
   if (pReq == NULL) {
     return NULL;
   }
 
-  strcpy(pReq->topic, pTopic->topicName);
+  /*strcpy(pReq->topic, pTopic->topicName);*/
-  strcpy(pReq->cgroup, tmq->groupId);
+  /*strcpy(pReq->cgroup, tmq->groupId);*/
+
+  int32_t tlen = strlen(tmq->groupId);
+  memcpy(pReq->subKey, tmq->groupId, tlen);
+  pReq->subKey[tlen] = TMQ_SEPARATOR;
+  strcpy(pReq->subKey + tlen + 1, pTopic->topicName);
 
   pReq->blockingTime = blockingTime;
   pReq->consumerId = tmq->consumerId;
@@ -1184,102 +1196,27 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
   pReq->reqId = generateRequestId();
 
   pReq->head.vgId = htonl(pVg->vgId);
-  pReq->head.contLen = htonl(sizeof(SMqPollReq));
+  pReq->head.contLen = htonl(sizeof(SMqPollReqV2));
   return pReq;
 }
 
 SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
   pRspObj->resType = RES_TYPE__TMQ;
-  pRspObj->topic = strdup(pWrapper->topicHandle->topicName);
+  strncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
-  pRspObj->resIter = -1;
   pRspObj->vgId = pWrapper->vgHandle->vgId;
-  SMqPollRspV2* pRsp = &pWrapper->msg;
+  pRspObj->resIter = -1;
-  int32_t       blockNum = taosArrayGetSize(pRsp->blockPos);
+  memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp));
-  pRspObj->res = taosArrayInit(blockNum, sizeof(SReqResultInfo));
+
-  for (int32_t i = 0; i < blockNum; i++) {
+  /*SRetrieveTableRsp* pRetrieve = taosArrayGetP(pWrapper->msg.blockData, 0);*/
-    int32_t            pos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
+  pRspObj->resInfo.totalRows = 0;
-    SRetrieveTableRsp* pRetrieve = POINTER_SHIFT(pRsp->blockData, pos);
+  pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-    SReqResultInfo     resInfo = {0};
+  setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
-    resInfo.totalRows = 0;
+
-    resInfo.precision = TSDB_TIME_PRECISION_MILLI;
+  taosFreeQitem(pWrapper);
-    setResSchemaInfo(&resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
-    setQueryResultFromRsp(&resInfo, pRetrieve, true);
-    taosArrayPush(pRspObj->res, &resInfo);
-  }
   return pRspObj;
 }
 
-#if 0
-tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
-  tmq_message_t* msg = NULL;
-  for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
-    SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
-      SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-      int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
-      /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
-      /*continue;*/
-      /*}*/
-      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
-      if (pReq == NULL) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
-        return NULL;
-      }
-
-      SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
-      if (pParam == NULL) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
-        return NULL;
-      }
-      pParam->tmq = tmq;
-      pParam->pVg = pVg;
-      pParam->epoch = tmq->epoch;
-      pParam->sync = 1;
-      pParam->msg = &msg;
-      tsem_init(&pParam->rspSem, 0, 0);
-
-      SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
-      if (sendInfo == NULL) {
-        return NULL;
-      }
-
-      sendInfo->msgInfo = (SDataBuf){
-          .pData = pReq,
-          .len = sizeof(SMqPollReq),
-          .handle = NULL,
-      };
-      sendInfo->requestId = generateRequestId();
-      sendInfo->requestObjRefId = 0;
-      sendInfo->param = pParam;
-      sendInfo->fp = tmqPollCb;
-      sendInfo->msgType = TDMT_VND_CONSUME;
-
-      int64_t transporterId = 0;
-      /*printf("send poll\n");*/
-      atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
-      pVg->pollCnt++;
-      tmq->pollCnt++;
-
-      tsem_wait(&pParam->rspSem);
-      tmq_message_t* nmsg = NULL;
-      while (1) {
-        taosReadQitem(tmq->mqueue, (void**)&nmsg);
-        if (nmsg == NULL) continue;
-        while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {
-          taosReadQitem(tmq->mqueue, (void**)&nmsg);
-        }
-        return nmsg;
-      }
-    }
-  }
-  return NULL;
-}
-#endif
-
 int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
   /*printf("call poll\n");*/
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
@@ -1301,7 +1238,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 #endif
       }
       atomic_store_32(&pVg->vgSkipCnt, 0);
-      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         /*tsem_post(&tmq->rspSem);*/
@@ -1332,7 +1269,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       sendInfo->msgInfo = (SDataBuf){
           .pData = pReq,
-          .len = sizeof(SMqPollReq),
+          .len = sizeof(SMqPollReqV2),
           .handle = NULL,
       };
       sendInfo->requestId = pReq->reqId;
@@ -1343,7 +1280,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
-      atomic_add_fetch_32(&tmq->waitingRequest, 1);
+      /*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/
       tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
                pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
@@ -1385,7 +1322,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
 
     if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
-      atomic_sub_fetch_32(&tmq->readyRequest, 1);
+      /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
       /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
       if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) {
         /*printf("epoch match\n");*/
@@ -1393,7 +1330,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
         pVg->currentOffset = pollRspWrapper->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        if (pollRspWrapper->msg.dataLen == 0) {
+        if (pollRspWrapper->msg.blockNum == 0) {
           taosFreeQitem(pollRspWrapper);
           rspWrapper = NULL;
           continue;
@@ -1449,7 +1386,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
   // TODO: put into another thread or delayed queue
   int64_t status = atomic_load_64(&tmq->status);
-  tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
+  while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) {
+    tscDebug("not ready, retry\n");
+    taosSsleep(1);
+  }
 
   rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
   if (rspObj) {
@@ -1607,16 +1547,6 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
 }
 #endif
 
-#if 0
-void tmq_message_destroy(tmq_message_t* tmq_message) {
-  if (tmq_message == NULL) return;
-  SMqPollRsp* pRsp = &tmq_message->msg;
-  tDeleteSMqConsumeRsp(pRsp);
-  /*taosMemoryFree(tmq_message);*/
-  taosFreeQitem(tmq_message);
-}
-#endif
-
 tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; }
 
 const char* tmq_err2str(tmq_resp_err_t err) {

source/client/test/tmqTest.cpp

@@ -108,7 +108,7 @@ TEST(testCase, tmq_subscribe_ctb_Test) {
 
   while (1) {
     tmq_message_t* msg = tmq_consumer_poll(tmq, 1000);
-    tmq_message_destroy(msg);
+    taos_free_result(msg);
     //printf("get msg\n");
     //if (msg == NULL) break;
   }
@@ -141,7 +141,7 @@ TEST(testCase, tmq_subscribe_stb_Test) {
       tmq_commit(tmq, NULL, 0);
     }
     //tmq_commit(tmq, NULL, 0);
-    tmq_message_destroy(msg);
+    taos_free_result(msg);
     //printf("get msg\n");
   }
 }

source/common/src/tdatablock.c

@@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
+    int32_t dataLen = varDataTLen(pData);
+    if(type == TSDB_DATA_TYPE_JSON) {
+      if(*pData == TSDB_DATA_TYPE_NULL) {
+        dataLen = 0;
+      }else if(*pData == TSDB_DATA_TYPE_NCHAR) {
+        dataLen = varDataTLen(pData+CHAR_BYTES);
+      }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+        dataLen = LONG_BYTES;
+      }else if(*pData == TSDB_DATA_TYPE_BOOL) {
+        dataLen = CHAR_BYTES;
+      }
+      dataLen += CHAR_BYTES;
+    }
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;
-    if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) {
+    if (pAttr->allocLen < pAttr->length + dataLen) {
       uint32_t newSize = pAttr->allocLen;
       if (newSize == 0) {
         newSize = 8;
       }
 
-      while (newSize < pAttr->length + varDataTLen(pData)) {
+      while (newSize < pAttr->length + dataLen) {
         newSize = newSize * 1.5;
       }
 
@@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
     uint32_t len = pColumnInfoData->varmeta.length;
     pColumnInfoData->varmeta.offset[currentRow] = len;
 
-    memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData));
+    memcpy(pColumnInfoData->pData + len, pData, dataLen);
-    pColumnInfoData->varmeta.length += varDataTLen(pData);
+    pColumnInfoData->varmeta.length += dataLen;
   } else {
     memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
   }

source/common/src/tglobal.c

@@ -30,7 +30,6 @@ char     tsLocalEp[TSDB_EP_LEN] = {0};  // Local End Point, hostname:port
 uint16_t tsServerPort = 6030;
 int32_t  tsVersion = 30000000;
 int32_t  tsStatusInterval = 1;  // second
-bool     tsEnableTelemetryReporting = false;
 
 // common
 int32_t tsRpcTimer = 300;
@@ -75,6 +74,12 @@ uint16_t tsMonitorPort = 6043;
 int32_t  tsMonitorMaxLogs = 100;
 bool     tsMonitorComp = false;
 
+// telem
+bool     tsEnableTelem = false;
+int32_t  tsTelemInterval = 86400;
+char     tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com";
+uint16_t tsTelemPort = 80;
+
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
  * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@@ -284,6 +289,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1;
   return 0;
 }
 
@@ -354,7 +360,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1;
   if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1;
-  if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1;
@@ -430,12 +435,17 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 1, 1024, 0) != 0) return -1;
 
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
-  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1;
   if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1;
   if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, 0) != 0) return -1;
 
+  if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, 0) != 0) return -1;
+  if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, 0) != 0) return -1;
+
   return 0;
 }
 
@@ -464,6 +474,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
   tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
   tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
   fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
+  fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32;
 }
 
 static int32_t taosSetClientCfg(SConfig *pCfg) {
@@ -528,7 +539,6 @@ static void taosSetSystemCfg(SConfig *pCfg) {
 static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
   tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
-  tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval;
   tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32;
   tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32;
   tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
@@ -572,6 +582,11 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
   tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
 
+  tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
+  tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32;
+  tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN);
+  tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
+
   if (tsQueryBufferSize >= 0) {
     tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
   }

source/common/src/tmsg.c

@@ -125,14 +125,14 @@ int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   return tlen;
 }
 
-void *taosDecodeSEpSet(void *buf, SEpSet *pEp) {
+void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) {
   buf = taosDecodeFixedI8(buf, &pEp->inUse);
   buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
     buf = taosDecodeFixedU16(buf, &pEp->eps[i].port);
     buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn);
   }
-  return buf;
+  return (void *)buf;
 }
 
 static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) {
@@ -1515,6 +1515,7 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
   if (tEncodeI8(&encoder, pReq->outputType) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->outputLen) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->bufSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->signature) < 0) return -1;
 
   int32_t codeSize = 0;
@@ -1554,6 +1555,7 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
   if (tDecodeI8(&decoder, &pReq->outputType) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->outputLen) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->bufSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1;
 
   int32_t codeSize = 0;
@@ -2672,6 +2674,10 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withTagSchema) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
@@ -2694,6 +2700,10 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withTagSchema) < 0) return -1;
   if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
   if (tDecodeI32(&decoder, &astLen) < 0) return -1;
 
@@ -2754,6 +2764,8 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
   if (tEncodeI32(&encoder, pReq->pid) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->app) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->passwd) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1;
   tEndEncode(&encoder);
 
@@ -2771,6 +2783,8 @@ int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
   if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->passwd) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->startTime) < 0) return -1;
   tEndDecode(&decoder);
 
@@ -3030,7 +3044,6 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
   return 0;
 }
 
-
 int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -3050,7 +3063,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq
     SReplica *pReplica = &pReq->replicas[i];
     if (tEncodeSReplica(&encoder, pReplica) < 0) return -1;
   }
-  
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -3083,7 +3096,6 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
   return 0;
 }
 
-
 int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -3597,8 +3609,6 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
 
@@ -3646,3 +3656,27 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
   taosMemoryFreeClear(pReq->sql);
   taosMemoryFreeClear(pReq->ast);
 }
+
+STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
+  STSchemaBuilder schemaBuilder = {0};
+  if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) {
+    return NULL;
+  }
+
+  for (int i = 0; i < nCols; i++) {
+    SSchema *schema = *pSchema + i;
+    if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) {
+      tdDestroyTSchemaBuilder(&schemaBuilder);
+      return NULL;
+    }
+  }
+
+  STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
+  if (pNSchema == NULL) {
+    tdDestroyTSchemaBuilder(&schemaBuilder);
+    return NULL;
+  }
+
+  tdDestroyTSchemaBuilder(&schemaBuilder);
+  return pNSchema;
+}

source/common/src/trow.c

@@ -24,9 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{
                                    },
                                    {
                                        // 1 bit
-                                       TD_VTYPE_NORM_BYTE_I,
+                                       TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I,
-                                       TD_VTYPE_NULL_BYTE_I,
+                                       TD_VTYPE_NULL_BYTE_I,  // padding
-                                       TD_VTYPE_NULL_BYTE_I, // padding
                                    }
 
 };
@@ -224,17 +223,40 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) {
  * @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker.
  *
  * @param srcBitmap
- * @param srcLen
+ * @param nBits
  * @param dstBitmap
  */
-void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap) {
+void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) {
   int32_t i = 0, j = 0;
+  int32_t nBytes = TD_BITMAP_BYTES(nBits);
+  int32_t nStrictBytes = nBits / 4;
+  int32_t nPartialBits = nBits - nStrictBytes * 4;
 
-  if (srcLen > 0) {
+  switch (nPartialBits) {
+    case 0:
+      // NOTHING TODO
+      break;
+    case 1: {
+      void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+      *(uint8_t *)lastByte &= 0xC0;
+    } break;
+    case 2: {
+      void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+      *(uint8_t *)lastByte &= 0xF0;
+    } break;
+    case 3: {
+      void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+      *(uint8_t *)lastByte &= 0xFC;
+    } break;
+    default:
+      ASSERT(0);
+  }
+
+  if (nBytes > 0) {
     dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4);
   }
 
-  while ((++i) < srcLen) {
+  while ((++i) < nBytes) {
     if ((i & 1) == 0) {
       dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4);
     } else {
@@ -381,17 +403,18 @@ STSRow *tdRowDup(STSRow *row) {
 }
 
 /**
- * @brief 
+ * @brief
- * 
+ *
- * @param pCol 
+ * @param pCol
- * @param valType 
+ * @param valType
- * @param val 
+ * @param val
- * @param numOfRows 
+ * @param numOfRows
- * @param maxPoints 
+ * @param maxPoints
  * @param bitmapMode  default is 0(2 bits), otherwise 1(1 bit)
- * @return int 
+ * @return int
  */
-int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, int8_t bitmapMode) {
+int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints,
+                         int8_t bitmapMode) {
   TASSERT(pCol != NULL);
 
   // Assume that the columns not specified during insert/upsert mean None.
@@ -426,7 +449,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int
     pCol->len += pCol->bytes;
   }
 #ifdef TD_SUPPORT_BITMAP
-    tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode);
+  tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode);
 #endif
   return 0;
 }
@@ -496,8 +519,9 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
     SKvRowIdx *pIdx = tdKvRowColIdxAt(pRow, rcol);
     int16_t    colIdx = -1;
     if (pIdx) {
-      colIdx = POINTER_DISTANCE(pRow->data, pIdx) / sizeof(SKvRowIdx);
+      colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx);
     }
+    TASSERT(colIdx >= 0);
     SCellVal sVal = {0};
     if (pIdx->colId == pDataCol->colId) {
       if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) {
@@ -537,7 +561,8 @@ int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCol
   return TSDB_CODE_SUCCESS;
 }
 
-int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, TDRowVerT maxVer) {
+int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull,
+                    TDRowVerT maxVer) {
   ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
   ASSERT(target->numOfCols == source->numOfCols);
   int offset = 0;
@@ -558,7 +583,8 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *
           if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) {
             TASSERT(0);
           }
-          tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
+          tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+                               target->bitmapMode);
         }
       }
       ++target->numOfRows;
@@ -605,7 +631,8 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
           if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
             TASSERT(0);
           }
-          tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
+          tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+                               target->bitmapMode);
         }
       }
 
@@ -621,12 +648,14 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
             TASSERT(0);
           }
           if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) {
-            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
+            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+                                 target->bitmapMode);
           } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
             if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
               TASSERT(0);
             }
-            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
+            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+                                 target->bitmapMode);
           } else if (target->cols[i].len > 0) {
             dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode);
           }

source/common/src/ttypes.c

@@ -379,7 +379,7 @@ static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *
   *maxIndex = 0;
 }
 
-tDataTypeDescriptor tDataTypes[15] = {
+tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
     {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
     {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
     {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint,
@@ -402,6 +402,7 @@ tDataTypeDescriptor tDataTypes[15] = {
     {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
     {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint,
      getStatics_u64},
+    {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {

source/common/src/tvariant.c

@@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
     }
     case TSDB_DATA_TYPE_BINARY: {
       pVar->pz = strndup(z, n);
-      pVar->nLen = strRmquote(pVar->pz, n);
+      //pVar->nLen = strRmquote(pVar->pz, n);
       break;
     }
     case TSDB_DATA_TYPE_TIMESTAMP: {

source/dnode/mgmt/implement/src/dmTransport.c

@@ -119,10 +119,10 @@ _OVER:
 }
 
 static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SDnodeTrans  *pTrans = &pDnode->trans;
+  SDnodeTrans * pTrans = &pDnode->trans;
   tmsg_t        msgType = pMsg->msgType;
   bool          isReq = msgType & 1u;
-  SMsgHandle   *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)];
+  SMsgHandle *  pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)];
   SMgmtWrapper *pWrapper = pHandle->pNdWrapper;
 
   if (msgType == TDMT_DND_SERVER_STATUS) {
@@ -443,7 +443,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s
   SAuthReq authReq = {0};
   tstrncpy(authReq.user, user, TSDB_USER_LEN);
   int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
-  void   *pReq = rpcMallocCont(contLen);
+  void *  pReq = rpcMallocCont(contLen);
   tSerializeSAuthReq(pReq, contLen, &authReq);
 
   SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};
@@ -523,4 +523,4 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) {
       .pWrapper = pWrapper,
   };
   return msgCb;
-}
+}

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -211,6 +211,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -252,7 +252,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -265,10 +265,11 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);  
   dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, DEFAULT_HANDLE);

source/dnode/mnode/impl/inc/mndConsumer.h

@@ -22,27 +22,30 @@
 extern "C" {
 #endif
 
-
 enum {
-  MQ_CONSUMER_STATUS__INIT = 1,
+  // MQ_CONSUMER_STATUS__INIT = 1,
-  MQ_CONSUMER_STATUS__IDLE,
+  MQ_CONSUMER_STATUS__MODIFY = 1,
-  MQ_CONSUMER_STATUS__ACTIVE,
+  MQ_CONSUMER_STATUS__MODIFY_IN_REB,
+  // MQ_CONSUMER_STATUS__IDLE,
+  MQ_CONSUMER_STATUS__READY,
   MQ_CONSUMER_STATUS__LOST,
-  MQ_CONSUMER_STATUS__MODIFY
+  MQ_CONSUMER_STATUS__LOST_IN_REB,
+  MQ_CONSUMER_STATUS__LOST_REBD,
 };
 
-
 int32_t mndInitConsumer(SMnode *pMnode);
 void    mndCleanupConsumer(SMnode *pMnode);
 
 SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId);
 void            mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
 
-SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup);
+SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup);
 
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
 SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
 
+int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndDef.h

@@ -88,6 +88,7 @@ typedef enum {
   TRN_TYPE_CREATE_STREAM = 1019,
   TRN_TYPE_DROP_STREAM = 1020,
   TRN_TYPE_ALTER_STREAM = 1021,
+  TRN_TYPE_CONSUMER_LOST = 1022,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -417,82 +418,6 @@ typedef struct {
   char    payload[];
 } SSysTableRetrieveObj;
 
-typedef struct {
-  int32_t vgId;  // -1 for unassigned
-  int32_t status;
-  int32_t epoch;
-  SEpSet  epSet;
-  int64_t oldConsumerId;
-  int64_t consumerId;  // -1 for unassigned
-  char*   qmsg;
-} SMqConsumerEp;
-
-static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pConsumerEp) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId);
-  tlen += taosEncodeFixedI32(buf, pConsumerEp->status);
-  tlen += taosEncodeFixedI32(buf, pConsumerEp->epoch);
-  tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet);
-  tlen += taosEncodeFixedI64(buf, pConsumerEp->oldConsumerId);
-  tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
-  tlen += taosEncodeString(buf, pConsumerEp->qmsg);
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
-  buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId);
-  buf = taosDecodeFixedI32(buf, &pConsumerEp->status);
-  buf = taosDecodeFixedI32(buf, &pConsumerEp->epoch);
-  buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet);
-  buf = taosDecodeFixedI64(buf, &pConsumerEp->oldConsumerId);
-  buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
-  buf = taosDecodeString(buf, &pConsumerEp->qmsg);
-  return buf;
-}
-
-static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) {
-  if (pConsumerEp) {
-    taosMemoryFreeClear(pConsumerEp->qmsg);
-  }
-}
-
-typedef struct {
-  int64_t consumerId;
-  SArray* vgInfo;  // SArray<SMqConsumerEp>
-} SMqSubConsumer;
-
-static FORCE_INLINE int32_t tEncodeSMqSubConsumer(void** buf, const SMqSubConsumer* pConsumer) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
-  int32_t sz = taosArrayGetSize(pConsumer->vgInfo);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp* pCEp = taosArrayGet(pConsumer->vgInfo, i);
-    tlen += tEncodeSMqConsumerEp(buf, pCEp);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqSubConsumer(void** buf, SMqSubConsumer* pConsumer) {
-  int32_t sz;
-  buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pConsumer->vgInfo = taosArrayInit(sz, sizeof(SMqConsumerEp));
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp consumerEp;
-    buf = tDecodeSMqConsumerEp(buf, &consumerEp);
-    taosArrayPush(pConsumer->vgInfo, &consumerEp);
-  }
-  return buf;
-}
-
-static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) {
-  if (pSubConsumer->vgInfo) {
-    taosArrayDestroyEx(pSubConsumer->vgInfo, (void (*)(void*))tDeleteSMqConsumerEp);
-    pSubConsumer->vgInfo = NULL;
-  }
-}
-
 typedef struct {
   char    key[TSDB_PARTITION_KEY_LEN];
   int64_t offset;
@@ -512,144 +437,20 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset)
 }
 
 typedef struct {
-  char    key[TSDB_SUBSCRIBE_KEY_LEN];
+  char    name[TSDB_TOPIC_FNAME_LEN];
-  int32_t status;
+  char    db[TSDB_DB_FNAME_LEN];
-  int32_t vgNum;
+  int64_t createTime;
-  SArray* consumers;      // SArray<SMqSubConsumer>
+  int64_t updateTime;
-  SArray* lostConsumers;  // SArray<SMqSubConsumer>
+  int64_t uid;
-  SArray* unassignedVg;   // SArray<SMqConsumerEp>
+  // TODO: use subDbUid
-} SMqSubscribeObj;
-
-static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
-  SMqSubscribeObj* pSub = taosMemoryCalloc(1, sizeof(SMqSubscribeObj));
-  if (pSub == NULL) {
-    return NULL;
-  }
-
-  pSub->consumers = taosArrayInit(0, sizeof(SMqSubConsumer));
-  if (pSub->consumers == NULL) {
-    goto _err;
-  }
-
-  pSub->lostConsumers = taosArrayInit(0, sizeof(SMqSubConsumer));
-  if (pSub->lostConsumers == NULL) {
-    goto _err;
-  }
-
-  pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp));
-  if (pSub->unassignedVg == NULL) {
-    goto _err;
-  }
-
-  pSub->key[0] = 0;
-  pSub->vgNum = 0;
-  pSub->status = 0;
-
-  return pSub;
-
-_err:
-  taosMemoryFreeClear(pSub->consumers);
-  taosMemoryFreeClear(pSub->lostConsumers);
-  taosMemoryFreeClear(pSub->unassignedVg);
-  taosMemoryFreeClear(pSub);
-  return NULL;
-}
-
-static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) {
-  int32_t tlen = 0;
-  tlen += taosEncodeString(buf, pSub->key);
-  tlen += taosEncodeFixedI32(buf, pSub->vgNum);
-  tlen += taosEncodeFixedI32(buf, pSub->status);
-  int32_t sz;
-
-  sz = taosArrayGetSize(pSub->consumers);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->consumers, i);
-    tlen += tEncodeSMqSubConsumer(buf, pSubConsumer);
-  }
-
-  sz = taosArrayGetSize(pSub->lostConsumers);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->lostConsumers, i);
-    tlen += tEncodeSMqSubConsumer(buf, pSubConsumer);
-  }
-
-  sz = taosArrayGetSize(pSub->unassignedVg);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedVg, i);
-    tlen += tEncodeSMqConsumerEp(buf, pCEp);
-  }
-
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) {
-  buf = taosDecodeStringTo(buf, pSub->key);
-  buf = taosDecodeFixedI32(buf, &pSub->vgNum);
-  buf = taosDecodeFixedI32(buf, &pSub->status);
-
-  int32_t sz;
-
-  buf = taosDecodeFixedI32(buf, &sz);
-  pSub->consumers = taosArrayInit(sz, sizeof(SMqSubConsumer));
-  if (pSub->consumers == NULL) {
-    return NULL;
-  }
-  for (int32_t i = 0; i < sz; i++) {
-    SMqSubConsumer subConsumer = {0};
-    buf = tDecodeSMqSubConsumer(buf, &subConsumer);
-    taosArrayPush(pSub->consumers, &subConsumer);
-  }
-
-  buf = taosDecodeFixedI32(buf, &sz);
-  pSub->lostConsumers = taosArrayInit(sz, sizeof(SMqSubConsumer));
-  if (pSub->lostConsumers == NULL) {
-    return NULL;
-  }
-  for (int32_t i = 0; i < sz; i++) {
-    SMqSubConsumer subConsumer = {0};
-    buf = tDecodeSMqSubConsumer(buf, &subConsumer);
-    taosArrayPush(pSub->lostConsumers, &subConsumer);
-  }
-
-  buf = taosDecodeFixedI32(buf, &sz);
-  pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp));
-  if (pSub->unassignedVg == NULL) {
-    return NULL;
-  }
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp consumerEp = {0};
-    buf = tDecodeSMqConsumerEp(buf, &consumerEp);
-    taosArrayPush(pSub->unassignedVg, &consumerEp);
-  }
-  return buf;
-}
-
-static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
-  if (pSub->consumers) {
-    // taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
-    //  taosArrayDestroy(pSub->consumers);
-    pSub->consumers = NULL;
-  }
-
-  if (pSub->unassignedVg) {
-    // taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
-    //  taosArrayDestroy(pSub->unassignedVg);
-    pSub->unassignedVg = NULL;
-  }
-}
-
-typedef struct {
-  char           name[TSDB_TOPIC_FNAME_LEN];
-  char           db[TSDB_DB_FNAME_LEN];
-  int64_t        createTime;
-  int64_t        updateTime;
-  int64_t        uid;
   int64_t        dbUid;
+  int64_t        subDbUid;
   int32_t        version;
+  int8_t         subType;  // db or table
+  int8_t         withTbName;
+  int8_t         withSchema;
+  int8_t         withTag;
+  int8_t         withTagSchema;
   SRWLatch       lock;
   int32_t        sqlLen;
   int32_t        astLen;
@@ -659,76 +460,115 @@ typedef struct {
   SSchemaWrapper schema;
 } SMqTopicObj;
 
+enum {
+  CONSUMER_UPDATE__TOUCH = 1,
+  CONSUMER_UPDATE__ADD,
+  CONSUMER_UPDATE__REMOVE,
+  CONSUMER_UPDATE__LOST,
+  CONSUMER_UPDATE__MODIFY,
+};
+
 typedef struct {
-  int64_t  consumerId;
+  int64_t consumerId;
-  int64_t  connId;
+  char    cgroup[TSDB_CGROUP_LEN];
-  SRWLatch lock;
+  int8_t  updateType;  // used only for update
-  char     cgroup[TSDB_CGROUP_LEN];
+  int32_t epoch;
-  SArray*  currentTopics;        // SArray<char*>
-  SArray*  recentRemovedTopics;  // SArray<char*>
-  int32_t  epoch;
-  // stat
-  int64_t pollCnt;
-  // status
   int32_t status;
-  // heartbeat from the consumer reset hbStatus to 0
+  // hbStatus is not applicable to serialization
-  // each checkConsumerAlive msg add hbStatus by 1
-  // if checkConsumerAlive > CONSUMER_REBALANCE_CNT, mask to lost
   int32_t hbStatus;
+  // lock is used for topics update
+  SRWLatch lock;
+  SArray*  currentTopics;     // SArray<char*>
+  SArray*  rebNewTopics;      // SArray<char*>
+  SArray*  rebRemovedTopics;  // SArray<char*>
 } SMqConsumerObj;
 
-static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) {
+SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);
-  int32_t sz;
+void            tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer);
-  int32_t tlen = 0;
+int32_t         tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer);
-  tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
+void*           tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer);
-  tlen += taosEncodeFixedI64(buf, pConsumer->connId);
-  tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
-  tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt);
-  tlen += taosEncodeFixedI32(buf, pConsumer->status);
-  tlen += taosEncodeString(buf, pConsumer->cgroup);
 
-  sz = taosArrayGetSize(pConsumer->currentTopics);
+typedef struct {
-  tlen += taosEncodeFixedI32(buf, sz);
+  int32_t vgId;
-  for (int32_t i = 0; i < sz; i++) {
+  char*   qmsg;
-    char* topic = taosArrayGetP(pConsumer->currentTopics, i);
+  SEpSet  epSet;
-    tlen += taosEncodeString(buf, topic);
+} SMqVgEp;
-  }
 
-  sz = taosArrayGetSize(pConsumer->recentRemovedTopics);
+SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp);
-  tlen += taosEncodeFixedI32(buf, sz);
+void     tDeleteSMqVgEp(SMqVgEp* pVgEp);
-  for (int32_t i = 0; i < sz; i++) {
+int32_t  tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp);
-    char* topic = taosArrayGetP(pConsumer->recentRemovedTopics, i);
+void*    tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp);
-    tlen += taosEncodeString(buf, topic);
-  }
-  return tlen;
-}
 
-static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) {
+typedef struct {
-  int32_t sz;
+  int64_t consumerId;  // -1 for unassigned
-  buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
+  SArray* vgs;         // SArray<SMqVgEp*>
-  buf = taosDecodeFixedI64(buf, &pConsumer->connId);
+} SMqConsumerEpInSub;
-  buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
-  buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt);
-  buf = taosDecodeFixedI32(buf, &pConsumer->status);
-  buf = taosDecodeStringTo(buf, pConsumer->cgroup);
 
-  buf = taosDecodeFixedI32(buf, &sz);
+SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub);
-  pConsumer->currentTopics = taosArrayInit(sz, sizeof(SMqConsumerObj));
+void                tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub);
-  for (int32_t i = 0; i < sz; i++) {
+int32_t             tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub);
-    char* topic;
+void*               tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub);
-    buf = taosDecodeString(buf, &topic);
-    taosArrayPush(pConsumer->currentTopics, &topic);
-  }
 
-  buf = taosDecodeFixedI32(buf, &sz);
+typedef struct {
-  pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(SMqConsumerObj));
+  char      key[TSDB_SUBSCRIBE_KEY_LEN];
-  for (int32_t i = 0; i < sz; i++) {
+  SRWLatch  lock;
-    char* topic;
+  int32_t   vgNum;
-    buf = taosDecodeString(buf, &topic);
+  int8_t    subType;
-    taosArrayPush(pConsumer->recentRemovedTopics, &topic);
+  int8_t    withTbName;
-  }
+  int8_t    withSchema;
-  return buf;
+  int8_t    withTag;
-}
+  int8_t    withTagSchema;
+  SHashObj* consumerHash;  // consumerId -> SMqConsumerEpInSub
+  // TODO put -1 into unassignVgs
+  // SArray*   unassignedVgs;
+} SMqSubscribeObj;
+
+SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
+SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub);
+void             tDeleteSubscribeObj(SMqSubscribeObj* pSub);
+int32_t          tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub);
+void*            tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub);
+
+typedef struct {
+  int32_t epoch;
+  SArray* consumers;  // SArray<SMqConsumerEpInSub*>
+} SMqSubActionLogEntry;
+
+SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+void                  tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+int32_t               tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry);
+void*                 tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry);
+
+typedef struct {
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
+  SArray* logs;  // SArray<SMqSubActionLogEntry*>
+} SMqSubActionLogObj;
+
+SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+void                tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+int32_t             tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog);
+void*               tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog);
+
+typedef struct {
+  const SMqSubscribeObj* pOldSub;
+  const SMqTopicObj*     pTopic;
+  const SMqRebSubscribe* pRebInfo;
+} SMqRebInputObj;
+
+typedef struct {
+  int64_t  oldConsumerId;
+  int64_t  newConsumerId;
+  SMqVgEp* pVgEp;
+} SMqRebOutputVg;
+
+typedef struct {
+  SArray*               rebVgs;            // SArray<SMqRebOutputVg>
+  SArray*               newConsumers;      // SArray<int64_t>
+  SArray*               removedConsumers;  // SArray<int64_t>
+  SArray*               touchedConsumers;  // SArray<int64_t>
+  SMqSubscribeObj*      pSub;
+  SMqSubActionLogEntry* pLogEntry;
+} SMqRebOutputObj;
 
 typedef struct {
   char     name[TSDB_TOPIC_FNAME_LEN];

source/dnode/mnode/impl/inc/mndInt.h

@@ -67,7 +67,6 @@ typedef struct {
 } SProfileMgmt;
 
 typedef struct {
-  bool     enable;
   SRWLatch lock;
   char     email[TSDB_FQDN_LEN];
 } STelemMgmt;

source/dnode/mnode/impl/inc/mndOffset.h

@@ -31,7 +31,7 @@ void          mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset);
 SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset);
 SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw);
 
-int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
+int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
 
 static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
   return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);

source/dnode/mnode/impl/inc/mndScheduler.h

@@ -29,6 +29,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
 
 int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
 
+int32_t mndConvertRSmaTask(const char* ast, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndSubscribe.h

@@ -26,9 +26,11 @@ int32_t mndInitSubscribe(SMnode *pMnode);
 void    mndCleanupSubscribe(SMnode *pMnode);
 
 SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName);
-SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char* key);
+SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key);
 void             mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
 
+int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndConsumer.c

@@ -19,8 +19,10 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndShow.h"
 #include "mndStb.h"
+#include "mndSubscribe.h"
 #include "mndTopic.h"
 #include "mndTrans.h"
 #include "mndUser.h"
@@ -28,9 +30,13 @@
 #include "tcompare.h"
 #include "tname.h"
 
-#define MND_CONSUMER_VER_NUMBER 1
+#define MND_CONSUMER_VER_NUMBER   1
 #define MND_CONSUMER_RESERVE_SIZE 64
 
+#define MND_CONSUMER_LOST_HB_CNT 3
+
+static int8_t mqInRebFlag = 0;
+
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
@@ -38,6 +44,11 @@ static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg);
 static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConsumer(SMnode *pMnode, void *pIter);
 
+static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
+static int32_t mndProcessAskEpReq(SNodeMsg *pMsg);
+static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
+static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg);
+
 int32_t mndInitConsumer(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_CONSUMER,
                      .keyType = SDB_KEY_INT64,
@@ -47,25 +58,397 @@ int32_t mndInitConsumer(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndConsumerActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
 
+  mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg);
   return sdbSetTable(pMnode->pSdb, table);
 }
 
 void mndCleanupConsumer(SMnode *pMnode) {}
 
-SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) {
+static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) {
-  SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
+  SMnode             *pMnode = pMsg->pNode;
-  if (pConsumer == NULL) {
+  SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont;
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
-    return NULL;
+  ASSERT(pConsumer);
+
+  SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
+  pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
+
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_LOST, &pMsg->rpcMsg);
+  if (pTrans == NULL) goto FAIL;
+  if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+FAIL:
+  // TODO delete consumer
+  mndTransDrop(pTrans);
+  return -1;
+}
+
+static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
+  SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1);
+  if (pRebSub == NULL) {
+    pRebSub = tNewSMqRebSubscribe(key);
+    if (pRebSub == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+    taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe));
+  }
+  return pRebSub;
+}
+
+static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
+  SMnode         *pMnode = pMsg->pNode;
+  SSdb           *pSdb = pMnode->pSdb;
+  SMqConsumerObj *pConsumer;
+  void           *pIter = NULL;
+
+  // rebalance cannot be parallel
+  int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1);
+  if (old != 0) {
+    mInfo("mq rebalance already in progress, do nothing");
+    return 0;
   }
 
-  pConsumer->recentRemovedTopics = taosArrayInit(1, sizeof(char *));
+  SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
-  pConsumer->epoch = 1;
+  pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
-  pConsumer->consumerId = consumerId;
+  // TODO set cleanfp
-  atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT);
+  pRebMsg->mqInReb = &mqInRebFlag;
-  strcpy(pConsumer->cgroup, cgroup);
+
-  taosInitRWLatch(&pConsumer->lock);
+  // iterate all consumers, find all modification
-  return pConsumer;
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
+    if (pIter == NULL) break;
+
+    int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
+    int32_t status = atomic_load_32(&pConsumer->status);
+    if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) {
+      SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg));
+
+      pLostMsg->consumerId = pConsumer->consumerId;
+      SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
+      pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_LOST;
+      pRpcMsg->pCont = pLostMsg;
+      pRpcMsg->contLen = sizeof(SMqConsumerLostMsg);
+      tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
+    }
+    if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) {
+      // do nothing
+    } else if (status == MQ_CONSUMER_STATUS__LOST) {
+      taosRLockLatch(&pConsumer->lock);
+      int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics);
+      for (int32_t i = 0; i < topicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
+      }
+      taosRUnLockLatch(&pConsumer->lock);
+    } else if (status == MQ_CONSUMER_STATUS__MODIFY) {
+      taosRLockLatch(&pConsumer->lock);
+      int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
+      for (int32_t i = 0; i < newTopicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId);
+      }
+
+      int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics);
+      for (int32_t i = 0; i < removedTopicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
+      }
+      taosRUnLockLatch(&pConsumer->lock);
+    } else {
+      // do nothing
+    }
+
+    mndReleaseConsumer(pMnode, pConsumer);
+  }
+
+  if (taosHashGetSize(pRebMsg->rebSubHash) != 0) {
+    mInfo("mq rebalance will be triggered");
+    SRpcMsg rpcMsg = {
+        .msgType = TDMT_MND_MQ_DO_REBALANCE,
+        .pCont = pRebMsg,
+        .contLen = sizeof(SMqDoRebalanceMsg),
+    };
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+  } else {
+    taosHashCleanup(pRebMsg->rebSubHash);
+    rpcFreeCont(pRebMsg);
+    mTrace("mq rebalance finished, no modification");
+    atomic_store_8(&mqInRebFlag, 0);
+  }
+  return 0;
+}
+
+static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
+  SMnode           *pMnode = pMsg->pNode;
+  SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
+  SMqCMGetSubEpRsp  rsp = {0};
+  int64_t           consumerId = be64toh(pReq->consumerId);
+  int32_t           epoch = ntohl(pReq->epoch);
+
+  SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId);
+  if (pConsumer == NULL) {
+    terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
+    return -1;
+  }
+
+  ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
+  /*int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);*/
+  atomic_store_32(&pConsumer->hbStatus, 0);
+
+  // 1. check consumer status
+  int32_t status = atomic_load_32(&pConsumer->status);
+
+  if (status == MQ_CONSUMER_STATUS__LOST) {
+    // TODO: recover consumer
+  }
+
+  if (status != MQ_CONSUMER_STATUS__READY) {
+    terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
+    return -1;
+  }
+
+  int32_t serverEpoch = atomic_load_32(&pConsumer->epoch);
+
+  // 2. check epoch, only send ep info when epoches do not match
+  if (epoch != serverEpoch) {
+    taosRLockLatch(&pConsumer->lock);
+    mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
+    int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics);
+
+    rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp));
+    if (rsp.topics == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      taosRUnLockLatch(&pConsumer->lock);
+      goto FAIL;
+    }
+
+    // handle all topic subscribed by the consumer
+    for (int32_t i = 0; i < numOfTopics; i++) {
+      char            *topic = taosArrayGetP(pConsumer->currentTopics, i);
+      SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
+
+      // txn guarantees pSub is created
+      ASSERT(pSub);
+      taosRLockLatch(&pSub->lock);
+
+      SMqSubTopicEp topicEp = {0};
+      strcpy(topicEp.topic, topic);
+
+      // 2.1 fetch topic schema
+      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+      ASSERT(pTopic);
+      taosRLockLatch(&pTopic->lock);
+      topicEp.schema.nCols = pTopic->schema.nCols;
+      topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema));
+      memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema));
+      taosRUnLockLatch(&pTopic->lock);
+      mndReleaseTopic(pMnode, pTopic);
+
+      // 2.2 iterate all vg assigned to the consumer of that topic
+      SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
+      int32_t             vgNum = taosArrayGetSize(pEpInSub->vgs);
+
+      topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
+      if (topicEp.vgs == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        taosRUnLockLatch(&pConsumer->lock);
+        goto FAIL;
+      }
+
+      for (int32_t j = 0; j < vgNum; j++) {
+        SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        char     offsetKey[TSDB_PARTITION_KEY_LEN];
+        mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
+        // 2.2.1 build vg ep
+        SMqSubVgEp vgEp = {
+            .epSet = pVgEp->epSet,
+            .vgId = pVgEp->vgId,
+            .offset = -1,
+        };
+
+        // 2.2.2 fetch vg offset
+        SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
+        if (pOffsetObj != NULL) {
+          vgEp.offset = atomic_load_64(&pOffsetObj->offset);
+          mndReleaseOffset(pMnode, pOffsetObj);
+        }
+        taosArrayPush(topicEp.vgs, &vgEp);
+      }
+      taosArrayPush(rsp.topics, &topicEp);
+
+      taosRUnLockLatch(&pSub->lock);
+      mndReleaseSubscribe(pMnode, pSub);
+    }
+    taosRUnLockLatch(&pConsumer->lock);
+  }
+  // encode rsp
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
+  void   *buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
+  ((SMqRspHead *)buf)->epoch = serverEpoch;
+  ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
+
+  void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+  tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
+
+  // release consumer and free memory
+  tDeleteSMqCMGetSubEpRsp(&rsp);
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  // send rsp
+  pMsg->pRsp = buf;
+  pMsg->rspLen = tlen;
+  return 0;
+FAIL:
+  tDeleteSMqCMGetSubEpRsp(&rsp);
+  mndReleaseConsumer(pMnode, pConsumer);
+  return -1;
+}
+
+int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer) {
+  SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
+  if (pCommitRaw == NULL) return -1;
+  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
+  if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
+
+static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
+  SMnode         *pMnode = pMsg->pNode;
+  char           *msgStr = pMsg->rpcMsg.pCont;
+  SCMSubscribeReq subscribe = {0};
+  tDeserializeSCMSubscribeReq(msgStr, &subscribe);
+  int64_t         consumerId = subscribe.consumerId;
+  char           *cgroup = subscribe.cgroup;
+  SMqConsumerObj *pConsumerOld = NULL;
+  SMqConsumerObj *pConsumerNew = NULL;
+
+  int32_t code = -1;
+  SArray *newSub = subscribe.topicNames;
+  taosArraySortString(newSub, taosArrayCompareString);
+
+  int32_t newTopicNum = taosArrayGetSize(newSub);
+  // check topic existance
+  for (int32_t i = 0; i < newTopicNum; i++) {
+    char        *topic = taosArrayGetP(newSub, i);
+    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+    if (pTopic == NULL) {
+      terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
+      goto SUBSCRIBE_OVER;
+    }
+    // TODO lock topic to prevent drop
+    mndReleaseTopic(pMnode, pTopic);
+  }
+
+  pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+  if (pConsumerOld == NULL) {
+    pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
+    /*pConsumerNew->waitingRebTopics = newSub;*/
+    pConsumerNew->rebNewTopics = newSub;
+    subscribe.topicNames = NULL;
+
+    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
+    if (pTrans == NULL) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+
+  } else {
+    /*taosRLockLatch(&pConsumerOld->lock);*/
+    int32_t status = atomic_load_32(&pConsumerOld->status);
+    if (status != MQ_CONSUMER_STATUS__READY) {
+      terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
+      goto SUBSCRIBE_OVER;
+    }
+
+    pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
+    if (pConsumerNew == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto SUBSCRIBE_OVER;
+    }
+    pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
+    /*pConsumerOld->waitingRebTopics = newSub;*/
+
+    int32_t oldTopicNum = 0;
+    if (pConsumerOld->currentTopics) {
+      oldTopicNum = taosArrayGetSize(pConsumerOld->currentTopics);
+    }
+
+    int32_t i = 0, j = 0;
+    while (i < oldTopicNum || j < newTopicNum) {
+      if (i >= oldTopicNum) {
+        char *newTopicCopy = strdup(taosArrayGetP(newSub, j));
+        taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
+        j++;
+        continue;
+      } else if (j >= newTopicNum) {
+        char *oldTopicCopy = strdup(taosArrayGetP(pConsumerOld->currentTopics, i));
+        taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
+        i++;
+        continue;
+      } else {
+        char *oldTopic = taosArrayGetP(pConsumerOld->currentTopics, i);
+        char *newTopic = taosArrayGetP(newSub, j);
+        int   comp = compareLenPrefixedStr(oldTopic, newTopic);
+        if (comp == 0) {
+          i++;
+          j++;
+          continue;
+        } else if (comp < 0) {
+          char *oldTopicCopy = strdup(oldTopic);
+          taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
+          i++;
+          continue;
+        } else {
+          char *newTopicCopy = strdup(newTopic);
+          taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
+          j++;
+          continue;
+        }
+      }
+    }
+
+    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
+    if (pTrans == NULL) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+  }
+
+  code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+
+SUBSCRIBE_OVER:
+  if (pConsumerOld) {
+    /*taosRUnLockLatch(&pConsumerOld->lock);*/
+    mndReleaseConsumer(pMnode, pConsumerOld);
+  }
+  if (pConsumerNew) {
+    tDeleteSMqConsumerObj(pConsumerNew);
+  }
+  // TODO: replace with destroy subscribe msg
+  if (subscribe.topicNames) taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
+  return code;
 }
 
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
@@ -154,15 +537,141 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
   mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId);
+  tDeleteSMqConsumerObj(pConsumer);
   return 0;
 }
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
-  /*taosWLockLatch(&pOldConsumer->lock);*/
-  atomic_add_fetch_32(&pOldConsumer->epoch, 1);
-  /*taosWUnLockLatch(&pOldConsumer->lock);*/
 
+  taosWLockLatch(&pOldConsumer->lock);
+
+  if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) {
+    ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
+    SArray *tmp = pOldConsumer->rebNewTopics;
+    pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics;
+    pNewConsumer->rebNewTopics = tmp;
+
+    tmp = pOldConsumer->rebRemovedTopics;
+    pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics;
+    pNewConsumer->rebRemovedTopics = tmp;
+
+    pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
+    int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
+    pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i));
+      taosArrayPush(pNewConsumer->rebRemovedTopics, &topic);
+    }
+    pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) {
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
+    ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);
+    ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
+
+    char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
+    // not exist in current topic
+#if 1
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
+      ASSERT(strcmp(topic, addedTopic) != 0);
+    }
+#endif
+
+    // remove from new topic
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      if (strcmp(addedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->rebNewTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+
+    // add to current topic
+    taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
+    taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString);
+    // set status
+    if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
+      }
+    } else {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
+      }
+    }
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
+    ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);
+    char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
+
+    // not exist in new topic
+#if 1
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      ASSERT(strcmp(topic, removedTopic) != 0);
+    }
+#endif
+
+    // remove from removed topic
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i);
+      if (strcmp(removedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->rebRemovedTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+
+    // remove from current topic
+    int32_t i = 0;
+    int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
+    for (i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
+      if (strcmp(removedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->currentTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+    // must find the topic
+    ASSERT(i < sz);
+
+    // set status
+    if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
+      }
+    } else {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
+      }
+    }
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  }
+
+  taosWUnLockLatch(&pOldConsumer->lock);
   return 0;
 }
 

source/dnode/mnode/impl/src/mndDef.c

@@ -14,6 +14,354 @@
  */
 
 #include "mndDef.h"
+#include "mndConsumer.h"
+
+SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) {
+  SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
+  if (pConsumer == NULL) {
+    return NULL;
+  }
+
+  pConsumer->consumerId = consumerId;
+  memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN);
+
+  pConsumer->epoch = 0;
+  pConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
+  pConsumer->hbStatus = 0;
+
+  taosInitRWLatch(&pConsumer->lock);
+
+  pConsumer->currentTopics = taosArrayInit(0, sizeof(void *));
+  pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
+  pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
+
+  if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) {
+    taosArrayDestroy(pConsumer->currentTopics);
+    taosArrayDestroy(pConsumer->rebNewTopics);
+    taosArrayDestroy(pConsumer->rebRemovedTopics);
+    taosMemoryFree(pConsumer);
+    return NULL;
+  }
+
+  return pConsumer;
+}
+
+void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
+  if (pConsumer->currentTopics) {
+    taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree);
+  }
+  if (pConsumer->rebNewTopics) {
+    taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree);
+  }
+  if (pConsumer->rebRemovedTopics) {
+    taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
+  }
+}
+
+int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
+  int32_t tlen = 0;
+  int32_t sz;
+  tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
+  tlen += taosEncodeString(buf, pConsumer->cgroup);
+  tlen += taosEncodeFixedI8(buf, pConsumer->updateType);
+  tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
+  tlen += taosEncodeFixedI32(buf, pConsumer->status);
+
+  // current topics
+  if (pConsumer->currentTopics) {
+    sz = taosArrayGetSize(pConsumer->currentTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->currentTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+  // reb new topics
+  if (pConsumer->rebNewTopics) {
+    sz = taosArrayGetSize(pConsumer->rebNewTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->rebNewTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+  // reb removed topics
+  if (pConsumer->rebRemovedTopics) {
+    sz = taosArrayGetSize(pConsumer->rebRemovedTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->rebRemovedTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+  return tlen;
+}
+
+void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
+  int32_t sz;
+  buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
+  buf = taosDecodeStringTo(buf, pConsumer->cgroup);
+  buf = taosDecodeFixedI8(buf, &pConsumer->updateType);
+  buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
+  buf = taosDecodeFixedI32(buf, &pConsumer->status);
+
+  // current topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->currentTopics, &topic);
+  }
+
+  // reb new topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->rebNewTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->rebNewTopics, &topic);
+  }
+
+  // reb removed topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->rebRemovedTopics, &topic);
+  }
+
+  return (void *)buf;
+}
+
+SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
+  SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
+  if (pVgEpNew == NULL) return NULL;
+  pVgEpNew->vgId = pVgEp->vgId;
+  pVgEpNew->qmsg = strdup(pVgEp->qmsg);
+  pVgEpNew->epSet = pVgEp->epSet;
+  return pVgEpNew;
+}
+
+void tDeleteSMqVgEp(SMqVgEp *pVgEp) { taosMemoryFree(pVgEp->qmsg); }
+
+int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
+  tlen += taosEncodeString(buf, pVgEp->qmsg);
+  tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
+  return tlen;
+}
+
+void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
+  buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
+  buf = taosDecodeString(buf, &pVgEp->qmsg);
+  buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
+  return (void *)buf;
+}
+
+SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) {
+  SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));
+  if (pEpInSubNew == NULL) return NULL;
+  pEpInSubNew->consumerId = pEpInSub->consumerId;
+  pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp);
+  return pEpInSubNew;
+}
+
+void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) {
+  taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp);
+}
+
+int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId);
+  int32_t sz = taosArrayGetSize(pEpInSub->vgs);
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i);
+    tlen += tEncodeSMqVgEp(buf, pVgEp);
+  }
+  /*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/
+  return tlen;
+}
+
+void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) {
+  buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId);
+  /*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  pEpInSub->vgs = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
+    buf = tDecodeSMqVgEp(buf, pVgEp);
+    taosArrayPush(pEpInSub->vgs, &pVgEp);
+  }
+
+  return (void *)buf;
+}
+
+SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
+  SMqSubscribeObj *pSubNew = taosMemoryCalloc(1, sizeof(SMqSubscribeObj));
+  if (pSubNew == NULL) return NULL;
+  memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN);
+  taosInitRWLatch(&pSubNew->lock);
+  pSubNew->vgNum = -1;
+  pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  // TODO set free fp
+  SMqConsumerEpInSub epInSub = {
+      .consumerId = -1,
+      .vgs = taosArrayInit(0, sizeof(void *)),
+  };
+  int64_t unexistKey = -1;
+  taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
+  return pSubNew;
+}
+
+SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
+  SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj));
+  if (pSubNew == NULL) return NULL;
+  memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
+  taosInitRWLatch(&pSubNew->lock);
+
+  pSubNew->subType = pSub->subType;
+  pSubNew->withTbName = pSub->withTbName;
+  pSubNew->withSchema = pSub->withSchema;
+  pSubNew->withTag = pSub->withTag;
+  pSubNew->withTagSchema = pSub->withTagSchema;
+
+  pSubNew->vgNum = pSub->vgNum;
+  pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  /*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/
+  void               *pIter = NULL;
+  SMqConsumerEpInSub *pEpInSub = NULL;
+  while (1) {
+    pIter = taosHashIterate(pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    pEpInSub = (SMqConsumerEpInSub *)pIter;
+    SMqConsumerEpInSub newEp = {
+        .consumerId = pEpInSub->consumerId,
+        .vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp),
+    };
+    taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub));
+  }
+  return pSubNew;
+}
+
+void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
+  /*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/
+  taosHashCleanup(pSub->consumerHash);
+}
+
+int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pSub->key);
+  tlen += taosEncodeFixedI32(buf, pSub->vgNum);
+  tlen += taosEncodeFixedI8(buf, pSub->subType);
+  tlen += taosEncodeFixedI8(buf, pSub->withTbName);
+  tlen += taosEncodeFixedI8(buf, pSub->withSchema);
+  tlen += taosEncodeFixedI8(buf, pSub->withTag);
+  tlen += taosEncodeFixedI8(buf, pSub->withTagSchema);
+
+  void   *pIter = NULL;
+  int32_t sz = taosHashGetSize(pSub->consumerHash);
+  tlen += taosEncodeFixedI32(buf, sz);
+
+  int32_t cnt = 0;
+  while (1) {
+    pIter = taosHashIterate(pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub);
+    cnt++;
+  }
+  ASSERT(cnt == sz);
+  /*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/
+  return tlen;
+}
+
+void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
+  //
+  buf = taosDecodeStringTo(buf, pSub->key);
+  buf = taosDecodeFixedI32(buf, &pSub->vgNum);
+  buf = taosDecodeFixedI8(buf, &pSub->subType);
+  buf = taosDecodeFixedI8(buf, &pSub->withTbName);
+  buf = taosDecodeFixedI8(buf, &pSub->withSchema);
+  buf = taosDecodeFixedI8(buf, &pSub->withTag);
+  buf = taosDecodeFixedI8(buf, &pSub->withTagSchema);
+
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+
+  pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  for (int32_t i = 0; i < sz; i++) {
+    /*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/
+    SMqConsumerEpInSub epInSub = {0};
+    buf = tDecodeSMqConsumerEpInSub(buf, &epInSub);
+    taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
+  }
+
+  /*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/
+  return (void *)buf;
+}
+
+SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+  SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
+  if (pEntryNew == NULL) return NULL;
+  pEntryNew->epoch = pEntry->epoch;
+  pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub);
+  return pEntryNew;
+}
+
+void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub);
+}
+
+int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pEntry->epoch);
+  tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
+  return tlen;
+}
+void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
+  buf = taosDecodeFixedI32(buf, &pEntry->epoch);
+  buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+  return (void *)buf;
+}
+
+SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+  SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
+  if (pLogNew == NULL) return pLogNew;
+  memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
+  pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub);
+  return pLogNew;
+}
+
+void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub);
+}
+
+int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pLog->key);
+  tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry);
+  return tlen;
+}
+
+void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
+  buf = taosDecodeStringTo(buf, pLog->key);
+  buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+  return (void *)buf;
+}
 
 int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   int32_t sz = 0;

source/dnode/mnode/impl/src/mndFunc.c

@@ -77,7 +77,9 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) {
   SDB_SET_INT64(pRaw, dataPos, pFunc->signature, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, _OVER)
-  SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER)
+  if (pFunc->commentSize > 0) {
+    SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER)
+  }
   SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER)
   SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER)
   SDB_SET_DATALEN(pRaw, dataPos, _OVER);
@@ -125,13 +127,18 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, _OVER)
 
-  pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize);
+  if (pFunc->commentSize > 0) {
-  pFunc->pCode = taosMemoryCalloc(1, pFunc->codeSize);
+    pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize);
-  if (pFunc->pComment == NULL || pFunc->pCode == NULL) {
+    if (pFunc->pComment == NULL) {
-    goto _OVER;
+      goto _OVER;
+    }
+    SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER)
   }
 
-  SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER)
+  pFunc->pCode = taosMemoryCalloc(1, pFunc->codeSize);
+  if (pFunc->pCode == NULL) {
+    goto _OVER;
+  }
   SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER)
   SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER)
 
@@ -192,16 +199,20 @@ static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCr
   func.outputLen = pCreate->outputLen;
   func.bufSize = pCreate->bufSize;
   func.signature = pCreate->signature;
-  func.commentSize = strlen(pCreate->pComment) + 1;
+  if (NULL != pCreate->pComment) {
-  func.codeSize = strlen(pCreate->pCode) + 1;
+    func.commentSize = strlen(pCreate->pComment) + 1;
-  func.pComment = taosMemoryMalloc(func.commentSize);
+    func.pComment = taosMemoryMalloc(func.commentSize);
+  }
+  func.codeSize = pCreate->codeLen;
   func.pCode = taosMemoryMalloc(func.codeSize);
   if (func.pCode == NULL || func.pCode == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto _OVER;
   }
 
-  memcpy(func.pComment, pCreate->pComment, func.commentSize);
+  if (func.commentSize > 0) {
+    memcpy(func.pComment, pCreate->pComment, func.commentSize);
+  }
   memcpy(func.pCode, pCreate->pCode, func.codeSize);
 
   pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_FUNC, &pReq->rpcMsg);
@@ -293,16 +304,6 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
-  if (createReq.pComment == NULL) {
-    terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT;
-    goto _OVER;
-  }
-
-  if (createReq.pComment[0] == 0) {
-    terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT;
-    goto _OVER;
-  }
-
   if (createReq.pCode == NULL) {
     terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
     goto _OVER;
@@ -440,14 +441,20 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) {
     funcInfo.signature = pFunc->signature;
     funcInfo.commentSize = pFunc->commentSize;
     funcInfo.codeSize = pFunc->codeSize;
-    funcInfo.pCode = taosMemoryCalloc(1, sizeof(funcInfo.codeSize));
+    funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize);
-    funcInfo.pComment = taosMemoryCalloc(1, sizeof(funcInfo.commentSize));
+    if (funcInfo.pCode == NULL) {
-    if (funcInfo.pCode == NULL || funcInfo.pComment == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       goto RETRIEVE_FUNC_OVER;
     }
-    memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize);
     memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize);
+    if (funcInfo.commentSize > 0) {
+      funcInfo.pComment = taosMemoryCalloc(1, funcInfo.commentSize);
+      if (funcInfo.pComment == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        goto RETRIEVE_FUNC_OVER;
+      }
+      memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize);
+    }
     taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo);
     mndReleaseFunc(pMnode, pFunc);
   }

source/dnode/mnode/impl/src/mndOffset.c

@@ -130,13 +130,12 @@ OFFSET_DECODE_OVER:
   return pRow;
 }
 
-int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
+int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
-  int32_t code = 0;
   int32_t sz = taosArrayGetSize(vgs);
   for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i);
+    int32_t      vgId = *(int32_t *)taosArrayGet(vgs, i);
-    SMqOffsetObj   offsetObj;
+    SMqOffsetObj offsetObj;
-    if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) {
+    if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, vgId) < 0) {
       return -1;
     }
     offsetObj.offset = -1;
@@ -170,13 +169,22 @@ static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) {
     if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
       return -1;
     }
+    bool          create = false;
     SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
-    ASSERT(pOffsetObj);
+    if (pOffsetObj == NULL) {
+      pOffsetObj = taosMemoryMalloc(sizeof(SMqOffset));
+      memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN);
+      create = true;
+    }
     pOffsetObj->offset = pOffset->offset;
     SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
     sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
     mndTransAppendRedolog(pTrans, pOffsetRaw);
-    mndReleaseOffset(pMnode, pOffsetObj);
+    if (create) {
+      taosMemoryFree(pOffsetObj);
+    } else {
+      mndReleaseOffset(pMnode, pOffsetObj);
+    }
   }
 
   if (mndTransPrepare(pMnode, pTrans) != 0) {
@@ -201,7 +209,7 @@ static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) {
 
 static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) {
   mTrace("offset:%s, perform update action", pOldOffset->key);
-  pOldOffset->offset = pNewOffset->offset;
+  atomic_store_64(&pOldOffset->offset, pNewOffset->offset);
   return 0;
 }
 

source/dnode/mnode/impl/src/mndProfile.c

@@ -24,20 +24,20 @@
 #include "version.h"
 
 typedef struct {
-  uint32_t    id;
+  uint32_t id;
-  int8_t      connType;
+  int8_t   connType;
-  char        user[TSDB_USER_LEN];
+  char     user[TSDB_USER_LEN];
-  char        app[TSDB_APP_NAME_LEN];  // app name that invokes taosc
+  char     app[TSDB_APP_NAME_LEN];  // app name that invokes taosc
-  int64_t     appStartTimeMs;          // app start time
+  int64_t  appStartTimeMs;          // app start time
-  int32_t     pid;                     // pid of app that invokes taosc
+  int32_t  pid;                     // pid of app that invokes taosc
-  uint32_t    ip;
+  uint32_t ip;
-  uint16_t    port;
+  uint16_t port;
-  int8_t      killed;
+  int8_t   killed;
-  int64_t     loginTimeMs;
+  int64_t  loginTimeMs;
-  int64_t     lastAccessTimeMs;
+  int64_t  lastAccessTimeMs;
-  uint64_t    killId;
+  uint64_t killId;
-  int32_t     numOfQueries;
+  int32_t  numOfQueries;
-  SArray     *pQueries;      //SArray<SQueryDesc>
+  SArray * pQueries;  // SArray<SQueryDesc>
 } SConnObj;
 
 static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port,
@@ -45,7 +45,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType
 static void      mndFreeConn(SConnObj *pConn);
 static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId);
 static void      mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
-static void     *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter);
+static void *    mndGetNextConn(SMnode *pMnode, SCacheIter *pIter);
 static void      mndCancelGetNextConn(SMnode *pMnode, void *pIter);
 static int32_t   mndProcessHeartBeatReq(SNodeMsg *pReq);
 static int32_t   mndProcessConnectReq(SNodeMsg *pReq);
@@ -71,9 +71,9 @@ int32_t mndInitProfile(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryReq);
   mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnReq);
 
-//  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns);
+  //  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn);
-//  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries);
+  //  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery);
 
   return 0;
@@ -91,7 +91,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType
                                int32_t pid, const char *app, int64_t startTime) {
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
-  char connStr[255] = {0};
+  char    connStr[255] = {0};
   int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app);
   int32_t connId = mndGenerateUid(connStr, len);
   if (startTime == 0) startTime = taosGetTimestampMs();
@@ -174,10 +174,10 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
 }
 
 static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
-  SMnode     *pMnode = pReq->pNode;
+  SMnode *    pMnode = pReq->pNode;
-  SUserObj   *pUser = NULL;
+  SUserObj *  pUser = NULL;
-  SDbObj     *pDb = NULL;
+  SDbObj *    pDb = NULL;
-  SConnObj   *pConn = NULL;
+  SConnObj *  pConn = NULL;
   int32_t     code = -1;
   SConnectReq connReq = {0};
   char        ip[30] = {0};
@@ -194,6 +194,11 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
     mError("user:%s, failed to login while acquire user since %s", pReq->user, terrstr());
     goto CONN_OVER;
   }
+  if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
+    mError("user:%s, failed to auth while acquire user\n %s \r\n %s", pReq->user, connReq.passwd, pUser->pass);
+    code = TSDB_CODE_RPC_AUTH_FAILURE;
+    goto CONN_OVER;
+  }
 
   if (connReq.db[0]) {
     char db[TSDB_DB_FNAME_LEN];
@@ -253,8 +258,8 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) {
 
   pConn->pQueries = pBasic->queryDesc;
   pBasic->queryDesc = NULL;
-  
+
-  pConn->numOfQueries =  pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0;
+  pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -324,9 +329,10 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) {
   return NULL;
 }
 
-static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, SClientHbBatchRsp *pBatchRsp) {
+static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq,
+                                        SClientHbBatchRsp *pBatchRsp) {
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
-  SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
+  SClientHbRsp  hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
 
   if (pHbReq->query) {
     SQueryHbReqBasic *pBasic = pHbReq->query;
@@ -335,8 +341,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
     rpcGetConnInfo(pMsg->handle, &connInfo);
 
     SConnObj *pConn = mndAcquireConn(pMnode, pBasic->connId);
-    if (pConn == NULL) {    
+    if (pConn == NULL) {
-      pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, pBasic->pid, pBasic->app, 0);
+      pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort,
+                            pBasic->pid, pBasic->app, 0);
       if (pConn == NULL) {
         mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr());
         return -1;
@@ -345,7 +352,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
       }
     } else if (pConn->killed) {
       mError("user:%s, conn:%u is already killed", connInfo.user, pConn->id);
-      mndReleaseConn(pMnode, pConn);      
+      mndReleaseConn(pMnode, pConn);
       terrno = TSDB_CODE_MND_INVALID_CONNECTION;
       return -1;
     }
@@ -369,8 +376,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
     }
 
     rspBasic->connId = pConn->id;
-    rspBasic->totalDnodes = 1;  //TODO
+    rspBasic->totalDnodes = 1;   // TODO
-    rspBasic->onlineDnodes = 1; //TODO
+    rspBasic->onlineDnodes = 1;  // TODO
     mndGetMnodeEpSet(pMnode, &rspBasic->epSet);
     mndReleaseConn(pMnode, pConn);
 
@@ -379,7 +386,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
 
   int32_t kvNum = taosHashGetSize(pHbReq->info);
   if (NULL == pHbReq->info || kvNum <= 0) {
-    taosArrayPush(pBatchRsp->rsps, &hbRsp);  
+    taosArrayPush(pBatchRsp->rsps, &hbRsp);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -396,7 +403,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
 
     switch (kv->key) {
       case HEARTBEAT_KEY_DBINFO: {
-        void   *rspMsg = NULL;
+        void *  rspMsg = NULL;
         int32_t rspLen = 0;
         mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen);
         if (rspMsg && rspLen > 0) {
@@ -406,7 +413,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
         break;
       }
       case HEARTBEAT_KEY_STBINFO: {
-        void   *rspMsg = NULL;
+        void *  rspMsg = NULL;
         int32_t rspLen = 0;
         mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen);
         if (rspMsg && rspLen > 0) {
@@ -457,7 +464,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
   taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
 
   int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp);
-  void   *buf = rpcMallocCont(tlen);
+  void *  buf = rpcMallocCont(tlen);
   tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp);
 
   int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps);
@@ -479,7 +486,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
-  SMnode       *pMnode = pReq->pNode;
+  SMnode *      pMnode = pReq->pNode;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
   SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@@ -513,7 +520,7 @@ static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
-  SMnode       *pMnode = pReq->pNode;
+  SMnode *      pMnode = pReq->pNode;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
   SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@@ -545,11 +552,11 @@ static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode   *pMnode = pReq->pNode;
+  SMnode *  pMnode = pReq->pNode;
   int32_t   numOfRows = 0;
   SConnObj *pConn = NULL;
   int32_t   cols = 0;
-  char     *pWrite;
+  char *    pWrite;
   char      ipStr[TSDB_IPv4ADDR_LEN + 6];
 
   if (pShow->pIter == NULL) {
@@ -604,8 +611,8 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int
 }
 
 static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode   *pMnode = pReq->pNode;
+  SMnode *pMnode = pReq->pNode;
-  int32_t   numOfRows = 0;
+  int32_t numOfRows = 0;
 #if 0
   SConnObj *pConn = NULL;
   int32_t   cols = 0;
@@ -703,7 +710,7 @@ static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, i
   }
 
   pShow->numOfRows += numOfRows;
-#endif  
+#endif
   return numOfRows;
 }
 

source/dnode/mnode/impl/src/mndScheduler.c

@@ -34,6 +34,54 @@
 
 extern bool tsStreamSchedV;
 
+int32_t mndConvertRSmaTask(const char* ast, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen) {
+  SNode*      pAst = NULL;
+  SQueryPlan* pPlan = NULL;
+  terrno = TSDB_CODE_SUCCESS;
+
+  if (nodesStringToNode(ast, &pAst) < 0) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    goto END;
+  }
+
+  SPlanContext cxt = {
+      .pAstRoot = pAst,
+      .topicQuery = false,
+      .streamQuery = true,
+      .rSmaQuery = true,
+      .triggerType = triggerType,
+      .watermark = watermark,
+  };
+  if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    goto END;
+  }
+
+  int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
+  if (levelNum != 1) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    goto END;
+  }
+  SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0);
+
+  int32_t opNum = LIST_LENGTH(inner->pNodeList);
+  if (opNum != 1) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    goto END;
+  }
+
+  SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
+  if (qSubPlanToString(plan, pStr, pLen) < 0) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    goto END;
+  }
+
+END:
+  if (pAst) nodesDestroyNode(pAst);
+  if (pPlan) nodesDestroyNode(pPlan);
+  return terrno;
+}
+
 int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) {
   SCoder encoder;
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
@@ -434,7 +482,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     return -1;
   }
 
-  ASSERT(pSub->vgNum == 0);
+  ASSERT(pSub->vgNum == -1);
+
+  pSub->vgNum = 0;
 
   int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
   if (levelNum != 1) {
@@ -453,6 +503,12 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
   }
   SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
 
+  int64_t             unexistKey = -1;
+  SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
+  ASSERT(pEpInSub);
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
   void* pIter = NULL;
   while (1) {
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@@ -466,24 +522,41 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     plan->execNode.nodeId = pVgroup->vgId;
     plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
+    SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
+    pVgEp->epSet = plan->execNode.epSet;
+    pVgEp->vgId = plan->execNode.nodeId;
+
+#if 0
     SMqConsumerEp consumerEp = {0};
     consumerEp.status = 0;
     consumerEp.consumerId = -1;
     consumerEp.epSet = plan->execNode.epSet;
     consumerEp.vgId = plan->execNode.nodeId;
-    
+#endif
-    mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId);
+
+    mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId);
 
     int32_t msgLen;
-    if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
+    if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) {
       sdbRelease(pSdb, pVgroup);
       qDestroyQueryPlan(pPlan);
       terrno = TSDB_CODE_QRY_INVALID_INPUT;
       return -1;
     }
-    taosArrayPush(pSub->unassignedVg, &consumerEp);
+    taosArrayPush(pEpInSub->vgs, &pVgEp);
+
+    ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
+    /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
   }
 
+  ASSERT(pEpInSub->vgs->size > 0);
+  pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
+
+  ASSERT(pEpInSub->vgs->size > 0);
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
   qDestroyQueryPlan(pPlan);
 
   return 0;

source/dnode/mnode/impl/src/mndStb.c

@@ -348,7 +348,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
   memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema));
   memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema));
   if (pNew->commentLen != 0) {
-    memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN);
+    memcpy(pOld->comment, pNew->comment, pNew->commentLen);
   }
   if (pNew->ast1Len != 0) {
     memcpy(pOld->pAst1, pNew->pAst1, pNew->ast1Len);

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -40,30 +40,30 @@ enum {
   MQ_SUBSCRIBE_STATUS__DELETED,
 };
 
-static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
-
 static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
 static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
 static int32_t  mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub);
 
-static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
+static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg);
-static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg);
-static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg);
 static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg);
-static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
-static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg);
-static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg);
-static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);
 
-static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
+static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
-                                      const SMqConsumerEp *pConsumerEp);
+  SSdbRaw *pRedoRaw = mndSubActionEncode(pSub);
+  if (pRedoRaw == NULL) return -1;
+  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
 
-static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
+static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
-                                      const char *topicName);
+  SSdbRaw *pCommitRaw = mndSubActionEncode(pSub);
-static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
+  if (pCommitRaw == NULL) return -1;
-                                       const char *oldTopicName);
+  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
+  if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
 
 int32_t mndInitSubscribe(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
@@ -74,242 +74,93 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndSubActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndSubActionDelete};
 
-  mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
+  mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq);
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);
-  mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
-  mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);
-  mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);
   return sdbSetTable(pMnode->pSdb, table);
 }
 
-static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) {
+static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) {
-  SMqSubscribeObj *pSub = tNewSubscribeObj();
+  SMqSubscribeObj *pSub = tNewSubscribeObj(subKey);
   if (pSub == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  char key[TSDB_SUBSCRIBE_KEY_LEN];
+  pSub->subType = pTopic->subType;
-  mndMakeSubscribeKey(key, cgroup, pTopic->name);
+  pSub->withTbName = pTopic->withTbName;
-  strcpy(pSub->key, key);
+  pSub->withSchema = pTopic->withSchema;
+  pSub->withTag = pTopic->withTag;
+  pSub->withTagSchema = pTopic->withTagSchema;
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
 
   if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
-    tDeleteSMqSubscribeObj(pSub);
+    tDeleteSubscribeObj(pSub);
     taosMemoryFree(pSub);
     return NULL;
   }
 
-  // TODO: disable alter subscribed table
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
   return pSub;
 }
 
-static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp,
+static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscribeObj *pSub,
-                                    const char *topicName) {
+                                    const SMqRebOutputVg *pRebVg) {
-  SMqMVRebReq req = {
+  SMqRebVgReq req = {0};
-      .vgId = pConsumerEp->vgId,
+  req.oldConsumerId = pRebVg->oldConsumerId;
-      .oldConsumerId = pConsumerEp->oldConsumerId,
+  req.newConsumerId = pRebVg->newConsumerId;
-      .newConsumerId = pConsumerEp->consumerId,
+  req.vgId = pRebVg->pVgEp->vgId;
-  };
+  req.qmsg = pRebVg->pVgEp->qmsg;
-  req.topic = strdup(topicName);
+  req.subType = pSub->subType;
+  req.withTbName = pSub->withTbName;
+  req.withSchema = pSub->withSchema;
+  req.withTag = pSub->withTag;
+  req.withTagSchema = pSub->withTagSchema;
+  strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
 
-  int32_t tlen = tEncodeSMqMVRebReq(NULL, &req);
+  int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req);
-  void   *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen);
+  void   *buf = taosMemoryMalloc(tlen);
   if (buf == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
   SMsgHead *pMsgHead = (SMsgHead *)buf;
 
-  pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen);
+  pMsgHead->contLen = htonl(tlen);
-  pMsgHead->vgId = htonl(pConsumerEp->vgId);
+  pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId);
 
   void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  tEncodeSMqMVRebReq(&abuf, &req);
+  tEncodeSMqRebVgReq(&abuf, &req);
-  taosMemoryFree(req.topic);
-
   *pBuf = buf;
   *pLen = tlen;
 
   return 0;
 }
 
-static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
+static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub,
-                                      const char *topicName) {
+                                        const SMqRebOutputVg *pRebVg) {
-  ASSERT(pConsumerEp->oldConsumerId != -1);
+  ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId);
 
   void   *buf;
   int32_t tlen;
-  if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp, topicName) < 0) {
+  if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg) < 0) {
     return -1;
   }
 
-  int32_t vgId = pConsumerEp->vgId;
+  int32_t vgId = pRebVg->pVgEp->vgId;
   SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
 
   STransAction action = {0};
   action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
   action.pCont = buf;
-  action.contLen = sizeof(SMsgHead) + tlen;
+  action.contLen = tlen;
-  action.msgType = TDMT_VND_MQ_REB;
+  action.msgType = TDMT_VND_MQ_VG_CHANGE;
 
   mndReleaseVgroup(pMnode, pVgObj);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(buf);
     return -1;
   }
-
-  return 0;
-}
-
-static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp,
-                                     const char *oldTopicName) {
-  SMqCancelConnReq req = {0};
-  req.consumerId = pConsumerEp->consumerId;
-  req.vgId = pConsumerEp->vgId;
-  req.epoch = pConsumerEp->epoch;
-  strcpy(req.topicName, oldTopicName);
-
-  int32_t tlen = tEncodeSMqCancelConnReq(NULL, &req);
-  void   *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen);
-  if (buf == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-  SMsgHead *pMsgHead = (SMsgHead *)buf;
-
-  pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen);
-  pMsgHead->vgId = htonl(pConsumerEp->vgId);
-  void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  tEncodeSMqCancelConnReq(&abuf, &req);
-  *pBuf = buf;
-  *pLen = tlen;
-  return 0;
-}
-
-static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
-                                       const char *oldTopicName) {
-  void   *buf;
-  int32_t tlen;
-  if (mndBuildCancelConnReq(&buf, &tlen, pConsumerEp, oldTopicName) < 0) {
-    return -1;
-  }
-
-  int32_t vgId = pConsumerEp->vgId;
-  SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
-
-  STransAction action = {0};
-  action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
-  action.pCont = buf;
-  action.contLen = sizeof(SMsgHead) + tlen;
-  action.msgType = TDMT_VND_MQ_CANCEL_CONN;
-
-  mndReleaseVgroup(pMnode, pVgObj);
-  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-    taosMemoryFree(buf);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
-  SMnode           *pMnode = pMsg->pNode;
-  SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
-  SMqCMGetSubEpRsp  rsp = {0};
-  int64_t           consumerId = be64toh(pReq->consumerId);
-  int32_t           epoch = ntohl(pReq->epoch);
-
-  SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId);
-  if (pConsumer == NULL) {
-    terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
-    return -1;
-  }
-  // TODO add lock
-  ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
-  int32_t serverEpoch = pConsumer->epoch;
-
-  // TODO
-  int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);
-  mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch,
-         hbStatus);
-  atomic_store_32(&pConsumer->hbStatus, 0);
-  /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/
-  /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/
-  /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/
-
-  strcpy(rsp.cgroup, pReq->cgroup);
-  if (epoch != serverEpoch) {
-    mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch,
-          serverEpoch);
-    mDebug("consumer %ld try r lock", consumerId);
-    taosRLockLatch(&pConsumer->lock);
-    mDebug("consumer %ld r locked", consumerId);
-    SArray *pTopics = pConsumer->currentTopics;
-    int32_t sz = taosArrayGetSize(pTopics);
-    rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
-    for (int32_t i = 0; i < sz; i++) {
-      char            *topicName = taosArrayGetP(pTopics, i);
-      SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topicName);
-      ASSERT(pSub);
-      int32_t csz = taosArrayGetSize(pSub->consumers);
-      // TODO: change to bsearch
-      for (int32_t j = 0; j < csz; j++) {
-        SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
-        if (consumerId == pSubConsumer->consumerId) {
-          int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
-          mInfo("topic %s has %d vg", topicName, serverEpoch);
-
-          SMqSubTopicEp topicEp;
-          strcpy(topicEp.topic, topicName);
-
-          SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topicName);
-          ASSERT(pTopic != NULL);
-          topicEp.schema = pTopic->schema;
-          mndReleaseTopic(pMnode, pTopic);
-
-          topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
-          for (int32_t k = 0; k < vgsz; k++) {
-            char           offsetKey[TSDB_PARTITION_KEY_LEN];
-            SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
-            SMqSubVgEp     vgEp = {
-                    .epSet = pConsumerEp->epSet,
-                    .vgId = pConsumerEp->vgId,
-                    .offset = -1,
-            };
-            mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId);
-            SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
-            if (pOffsetObj != NULL) {
-              vgEp.offset = pOffsetObj->offset;
-              mndReleaseOffset(pMnode, pOffsetObj);
-            }
-            taosArrayPush(topicEp.vgs, &vgEp);
-          }
-          taosArrayPush(rsp.topics, &topicEp);
-          break;
-        }
-      }
-      mndReleaseSubscribe(pMnode, pSub);
-    }
-    taosRUnLockLatch(&pConsumer->lock);
-    mDebug("consumer %ld r unlock", consumerId);
-  }
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
-  void   *buf = rpcMallocCont(tlen);
-  if (buf == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-  ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
-  ((SMqRspHead *)buf)->epoch = serverEpoch;
-  ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
-
-  void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
-  tDeleteSMqCMGetSubEpRsp(&rsp);
-  mndReleaseConsumer(pMnode, pConsumer);
-  pMsg->pRsp = buf;
-  pMsg->rspLen = tlen;
   return 0;
 }
 
@@ -337,82 +188,298 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
   return pRebSub;
 }
 
-static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
+static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
-  SMnode            *pMnode = pMsg->pNode;
+  if (pInput->pTopic != NULL) {
-  SSdb              *pSdb = pMnode->pSdb;
+    // create subscribe
-  SMqConsumerObj    *pConsumer;
+    pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key);
-  void              *pIter = NULL;
+    ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1);
-  SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
-  pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
-
-  while (1) {
-    pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
-    if (pIter == NULL) break;
-    int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
-    if (hbStatus > MND_SUBSCRIBE_REBALANCE_CNT) {
-      int32_t old =
-          atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST);
-      if (old == MQ_CONSUMER_STATUS__ACTIVE) {
-        // get all topics of that topic
-        int32_t sz = taosArrayGetSize(pConsumer->currentTopics);
-        for (int32_t i = 0; i < sz; i++) {
-          char *topic = taosArrayGetP(pConsumer->currentTopics, i);
-          char  key[TSDB_SUBSCRIBE_KEY_LEN];
-          mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
-          SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
-          taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId);
-        }
-      }
-    }
-    int32_t status = atomic_load_32(&pConsumer->status);
-    if (status == MQ_CONSUMER_STATUS__INIT || status == MQ_CONSUMER_STATUS__MODIFY) {
-      SArray *rebSubs;
-      if (status == MQ_CONSUMER_STATUS__INIT) {
-        rebSubs = pConsumer->currentTopics;
-      } else {
-        rebSubs = pConsumer->recentRemovedTopics;
-      }
-      int32_t sz = taosArrayGetSize(rebSubs);
-      for (int32_t i = 0; i < sz; i++) {
-        char *topic = taosArrayGetP(rebSubs, i);
-        char  key[TSDB_SUBSCRIBE_KEY_LEN];
-        mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
-        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
-        if (status == MQ_CONSUMER_STATUS__INIT) {
-          taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId);
-        } else if (status == MQ_CONSUMER_STATUS__MODIFY) {
-          taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
-        }
-      }
-      if (status == MQ_CONSUMER_STATUS__MODIFY) {
-        int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics);
-        for (int32_t i = 0; i < removeSz; i++) {
-          char *topicName = taosArrayGetP(pConsumer->recentRemovedTopics, i);
-          taosMemoryFree(topicName);
-        }
-        taosArrayClear(pConsumer->recentRemovedTopics);
-      }
-    }
-  }
-  if (taosHashGetSize(pRebMsg->rebSubHash) != 0) {
-    mInfo("mq rebalance will be triggered");
-    SRpcMsg rpcMsg = {
-        .msgType = TDMT_MND_MQ_DO_REBALANCE,
-        .pCont = pRebMsg,
-        .contLen = sizeof(SMqDoRebalanceMsg),
-    };
-    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
   } else {
-    taosHashCleanup(pRebMsg->rebSubHash);
+    pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub);
-    rpcFreeCont(pRebMsg);
   }
+  int32_t totalVgNum = pOutput->pSub->vgNum;
+
+  mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum);
+
+  // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
+  SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+
+  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
+  // 2. check and get actual removed consumers, put their vg into hash
+  int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+  int32_t actualRemoved = 0;
+  for (int32_t i = 0; i < removedNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    if (pEpInSub) {
+      ASSERT(consumerId == pEpInSub->consumerId);
+      actualRemoved++;
+      int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+      for (int32_t j = 0; j < consumerVgNum; j++) {
+        SMqVgEp       *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        SMqRebOutputVg outputVg = {
+            .oldConsumerId = consumerId,
+            .newConsumerId = -1,
+            .pVgEp = pVgEp,
+        };
+        taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+      }
+      taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+      // put into removed
+      taosArrayPush(pOutput->removedConsumers, &consumerId);
+    }
+  }
+  ASSERT(removedNum == actualRemoved);
+  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
+
+  // if previously no consumer, there are vgs not assigned
+  {
+    int64_t             unexistKey = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    for (int32_t i = 0; i < consumerVgNum; i++) {
+      SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+      SMqRebOutputVg rebOutput = {
+          .oldConsumerId = -1,
+          .newConsumerId = -1,
+          .pVgEp = pVgEp,
+      };
+      taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
+    }
+  }
+
+  // 3. calc vg number of each consumer
+  int32_t oldSz = 0;
+  if (pInput->pOldSub) {
+    oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1;
+  }
+  int32_t afterRebConsumerNum =
+      oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+  int32_t minVgCnt = 0;
+  int32_t imbConsumerNum = 0;
+  // calc num
+  if (afterRebConsumerNum) {
+    minVgCnt = totalVgNum / afterRebConsumerNum;
+    imbConsumerNum = totalVgNum % afterRebConsumerNum;
+  }
+
+  // 4. first scan: remove consumer more than wanted, put to remove hash
+  int32_t imbCnt = 0;
+  void   *pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    if (pEpInSub->consumerId == -1) continue;
+    ASSERT(pEpInSub->consumerId > 0);
+    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    // all old consumers still existing are touched
+    // TODO optimize: touch only consumer whose vgs changed
+    taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId);
+    if (consumerVgNum > minVgCnt) {
+      if (imbCnt < imbConsumerNum) {
+        if (consumerVgNum == minVgCnt + 1) {
+          continue;
+        } else {
+          // pop until equal minVg + 1
+          while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) {
+            SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+            SMqRebOutputVg outputVg = {
+                .oldConsumerId = pEpInSub->consumerId,
+                .newConsumerId = -1,
+                .pVgEp = pVgEp,
+            };
+            taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+          }
+          imbCnt++;
+        }
+      } else {
+        // pop until equal minVg
+        while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) {
+          SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+          SMqRebOutputVg outputVg = {
+              .oldConsumerId = pEpInSub->consumerId,
+              .newConsumerId = -1,
+              .pVgEp = pVgEp,
+          };
+          taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+        }
+      }
+    }
+  }
+
+  // 5. add new consumer into sub
+  {
+    int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers);
+    for (int32_t i = 0; i < consumerNum; i++) {
+      int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
+      ASSERT(consumerId > 0);
+      SMqConsumerEpInSub newConsumerEp;
+      newConsumerEp.consumerId = consumerId;
+      newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
+      taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp,
+                  sizeof(SMqConsumerEpInSub));
+      /*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
+      /*ASSERT(pTestNew->consumerId == consumerId);*/
+      /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/
+      taosArrayPush(pOutput->newConsumers, &consumerId);
+    }
+  }
+
+  // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer.
+  // All related vg should be put into rebVgs
+  SMqRebOutputVg *pRebVg = NULL;
+  void           *pRemovedIter = NULL;
+  pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    if (pEpInSub->consumerId == -1) continue;
+    ASSERT(pEpInSub->consumerId > 0);
+
+    // push until equal minVg
+    while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) {
+      // iter hash and find one vg
+      pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+      ASSERT(pRemovedIter);
+      pRebVg = (SMqRebOutputVg *)pRemovedIter;
+      // push
+      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pEpInSub->consumerId;
+      taosArrayPush(pOutput->rebVgs, pRebVg);
+    }
+  }
+
+  // 7. handle unassigned vg
+  if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) {
+    // if has consumer, assign all left vg
+    while (1) {
+      pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+      if (pRemovedIter == NULL) break;
+      pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+      ASSERT(pIter);
+      pRebVg = (SMqRebOutputVg *)pRemovedIter;
+      SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+      if (pEpInSub->consumerId == -1) continue;
+      ASSERT(pEpInSub->consumerId > 0);
+      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pEpInSub->consumerId;
+      taosArrayPush(pOutput->rebVgs, pRebVg);
+    }
+  } else {
+    // if all consumer is removed, put all vg into unassigned
+    int64_t             unexistKey = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    ASSERT(pEpInSub->consumerId == -1);
+
+    pIter = NULL;
+    SMqRebOutputVg *pRebOutput = NULL;
+    while (1) {
+      pIter = taosHashIterate(pHash, pIter);
+      if (pIter == NULL) break;
+      pRebOutput = (SMqRebOutputVg *)pIter;
+      ASSERT(pRebOutput->newConsumerId == -1);
+      taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp);
+      taosArrayPush(pOutput->rebVgs, pRebOutput);
+    }
+  }
+
+  // 8. generate logs
+
+  // 9. clear
+  taosHashCleanup(pHash);
+
   return 0;
 }
 
-static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
+static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebOutputObj *pOutput) {
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg);
+  if (pTrans == NULL) {
+    return -1;
+  }
+  // make txn:
+  // 1. redo action: action to all vg
+  const SArray *rebVgs = pOutput->rebVgs;
+  int32_t       vgNum = taosArrayGetSize(rebVgs);
+  for (int32_t i = 0; i < vgNum; i++) {
+    SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i);
+    if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg) < 0) {
+      goto REB_FAIL;
+    }
+  }
+
+  // 2. redo log: subscribe and vg assignment
+  // subscribe
+  if (mndSetSubRedoLogs(pMnode, pTrans, pOutput->pSub) != 0) {
+    goto REB_FAIL;
+  }
+
+  // 3. commit log: consumer to update status and epoch
+  // 3.1 set touched consumer
+  int32_t consumerNum = taosArrayGetSize(pOutput->touchedConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t         consumerId = *(int64_t *)taosArrayGet(pOutput->touchedConsumers, i);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH;
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+  // 3.2 set new consumer
+  consumerNum = taosArrayGetSize(pOutput->newConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__ADD;
+    char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
+    char  cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    taosArrayPush(pConsumerNew->rebNewTopics, &topic);
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+
+  // 3.3 set removed consumer
+  consumerNum = taosArrayGetSize(pOutput->removedConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE;
+    char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
+    char  cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    taosArrayPush(pConsumerNew->rebRemovedTopics, &topic);
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+  // 4. commit log: modification log
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+
+REB_FAIL:
+  mndTransDrop(pTrans);
+  return -1;
+}
+
+static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
   SMnode            *pMnode = pMsg->pNode;
   SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont;
-  STrans            *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg);
   void              *pIter = NULL;
 
   mInfo("mq rebalance start");
@@ -420,192 +487,52 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
   while (1) {
     pIter = taosHashIterate(pReq->rebSubHash, pIter);
     if (pIter == NULL) break;
+    SMqRebInputObj rebInput = {0};
+
+    SMqRebOutputObj rebOutput = {0};
+    rebOutput.newConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.removedConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.touchedConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg));
+
     SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter;
     SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
-    taosMemoryFreeClear(pRebSub->key);
 
-    mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum,
+    if (pSub == NULL) {
-          (int32_t)taosArrayGetSize(pSub->unassignedVg));
+      // split sub key and extract topic
-
+      char topic[TSDB_TOPIC_FNAME_LEN];
-    // remove lost consumer
+      char cgroup[TSDB_CGROUP_LEN];
-    for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) {
+      mndSplitSubscribeKey(pRebSub->key, topic, cgroup);
-      int64_t lostConsumerId = *(int64_t *)taosArrayGet(pRebSub->lostConsumers, i);
+      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
-
+      ASSERT(pTopic);
-      mInfo("mq remove lost consumer %" PRId64 "", lostConsumerId);
+      taosRLockLatch(&pTopic->lock);
-
+      rebInput.pTopic = pTopic;
-      for (int32_t j = 0; j < taosArrayGetSize(pSub->consumers); j++) {
-        SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
-        if (pSubConsumer->consumerId == lostConsumerId) {
-          taosArrayAddAll(pSub->unassignedVg, pSubConsumer->vgInfo);
-          taosArrayPush(pSub->lostConsumers, pSubConsumer);
-          taosArrayRemove(pSub->consumers, j);
-          break;
-        }
-      }
     }
 
-    // calculate rebalance
+    rebInput.pRebInfo = pRebSub;
-    int32_t consumerNum = taosArrayGetSize(pSub->consumers);
+    rebInput.pOldSub = pSub;
-    if (consumerNum != 0) {
-      int32_t vgNum = pSub->vgNum;
-      int32_t vgEachConsumer = vgNum / consumerNum;
-      int32_t imbalanceVg = vgNum % consumerNum;
 
-      // iterate all consumers, set unassignedVgStash
+    // TODO replace assert with error check
-      for (int32_t i = 0; i < consumerNum; i++) {
+    ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
-        SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
+    // if add more consumer to balanced subscribe,
-        int32_t         vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
+    // possibly no vg is changed
-        int32_t         vgThisConsumerAfterRb;
+    /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
-        if (i < imbalanceVg)
+    ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
-          vgThisConsumerAfterRb = vgEachConsumer + 1;
-        else
-          vgThisConsumerAfterRb = vgEachConsumer;
 
-        mInfo("mq consumer:%" PRId64 ", connectted vgroup number change from %d to %d", pSubConsumer->consumerId,
+    if (rebInput.pTopic) {
-              vgThisConsumerBeforeRb, vgThisConsumerAfterRb);
+      SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic;
-
+      taosRUnLockLatch(&pTopic->lock);
-        while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) {
+      mndReleaseTopic(pMnode, pTopic);
-          SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo);
+    } else {
-          ASSERT(pConsumerEp != NULL);
+      mndReleaseSubscribe(pMnode, pSub);
-          ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId);
-          taosArrayPush(pSub->unassignedVg, pConsumerEp);
-          mDebug("mq rebalance: vg %d push to unassignedVg", pConsumerEp->vgId);
-        }
-
-        SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
-        mDebug("consumer %ld try w lock", pRebConsumer->consumerId);
-        taosWLockLatch(&pRebConsumer->lock);
-        mDebug("consumer %ld w locked", pRebConsumer->consumerId);
-        int32_t status = atomic_load_32(&pRebConsumer->status);
-        if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb ||
-            (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) ||
-            (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) {
-          /*if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) {*/
-          /*pRebConsumer->epoch++;*/
-          /*}*/
-          if (vgThisConsumerAfterRb != 0) {
-            atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
-          } else {
-            atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE);
-          }
-
-          mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status,
-                pRebConsumer->status);
-
-          SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
-          sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-          mndTransAppendCommitlog(pTrans, pConsumerRaw);
-        }
-        taosWUnLockLatch(&pRebConsumer->lock);
-        mDebug("consumer %ld w unlock", pRebConsumer->consumerId);
-        mndReleaseConsumer(pMnode, pRebConsumer);
-      }
-
-      // assign to vgroup
-      if (taosArrayGetSize(pSub->unassignedVg) != 0) {
-        for (int32_t i = 0; i < consumerNum; i++) {
-          SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
-          int32_t         vgThisConsumerAfterRb;
-          if (i < imbalanceVg)
-            vgThisConsumerAfterRb = vgEachConsumer + 1;
-          else
-            vgThisConsumerAfterRb = vgEachConsumer;
-
-          while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) {
-            SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg);
-            mDebug("mq rebalance: vg %d pop from unassignedVg", pConsumerEp->vgId);
-            ASSERT(pConsumerEp != NULL);
-
-            pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
-            pConsumerEp->consumerId = pSubConsumer->consumerId;
-            // TODO
-            pConsumerEp->epoch = 0;
-            taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
-
-            char topic[TSDB_TOPIC_FNAME_LEN];
-            char cgroup[TSDB_CGROUP_LEN];
-            mndSplitSubscribeKey(pSub->key, topic, cgroup);
-            if (pConsumerEp->oldConsumerId == -1) {
-              SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
-
-              mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId,
-                    topic, pConsumerEp->consumerId, cgroup);
-
-              mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
-              mndReleaseTopic(pMnode, pTopic);
-            } else {
-              mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "",
-                    pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId);
-
-              mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, topic);
-            }
-          }
-        }
-      }
-      ASSERT(taosArrayGetSize(pSub->unassignedVg) == 0);
-
-      // TODO: log rebalance statistics
-      SSdbRaw *pSubRaw = mndSubActionEncode(pSub);
-      sdbSetRawStatus(pSubRaw, SDB_STATUS_READY);
-      mndTransAppendRedolog(pTrans, pSubRaw);
     }
-    mndReleaseSubscribe(pMnode, pSub);
-  }
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("mq-rebalance-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
-    taosHashCleanup(pReq->rebSubHash);
-    mndTransDrop(pTrans);
-    return -1;
   }
 
+  // reset flag
+  atomic_store_8(pReq->mqInReb, 0);
+  mInfo("mq rebalance completed successfully");
   taosHashCleanup(pReq->rebSubHash);
-  mndTransDrop(pTrans);
-  return 0;
-}
 
-static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
-                                      const SMqConsumerEp *pConsumerEp) {
-  ASSERT(pConsumerEp->oldConsumerId == -1);
-  int32_t vgId = pConsumerEp->vgId;
-
-  SMqSetCVgReq req = {
-      .vgId = vgId,
-      .consumerId = pConsumerEp->consumerId,
-      .sql = pTopic->sql,
-      .physicalPlan = pTopic->physicalPlan,
-      .qmsg = pConsumerEp->qmsg,
-  };
-
-  strcpy(req.cgroup, cgroup);
-  strcpy(req.topicName, pTopic->name);
-  int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
-  void   *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen);
-  if (buf == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  SMsgHead *pMsgHead = (SMsgHead *)buf;
-
-  pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen);
-  pMsgHead->vgId = htonl(vgId);
-
-  void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  tEncodeSMqSetCVgReq(&abuf, &req);
-
-  SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
-
-  STransAction action = {0};
-  action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
-  action.pCont = buf;
-  action.contLen = sizeof(SMsgHead) + tlen;
-  action.msgType = TDMT_VND_MQ_SET_CONN;
-
-  mndReleaseVgroup(pMnode, pVgObj);
-  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-    taosMemoryFree(buf);
-    return -1;
-  }
   return 0;
 }
 
@@ -697,16 +624,23 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) {
 
 static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) {
   mTrace("subscribe:%s, perform delete action", pSub->key);
-  tDeleteSMqSubscribeObj(pSub);
+  tDeleteSubscribeObj(pSub);
   return 0;
 }
 
 static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) {
   mTrace("subscribe:%s, perform update action", pOldSub->key);
+  taosWLockLatch(&pOldSub->lock);
+
+  SHashObj *tmp = pOldSub->consumerHash;
+  pOldSub->consumerHash = pNewSub->consumerHash;
+  pNewSub->consumerHash = tmp;
+
+  taosWUnLockLatch(&pOldSub->lock);
   return 0;
 }
 
-static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
+int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
   int32_t tlen = strlen(cgroup);
   memcpy(key, cgroup, tlen);
   key[tlen] = TMQ_SEPARATOR;
@@ -739,6 +673,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) {
   sdbRelease(pSdb, pSub);
 }
 
+#if 0
 static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   SMnode         *pMnode = pMsg->pNode;
   char           *msgStr = pMsg->rpcMsg.pCont;
@@ -901,6 +836,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer);
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
+#endif
 
 static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) {
   mndTransProcessRsp(pRsp);

source/dnode/mnode/impl/src/mndTelem.c

@@ -21,9 +21,6 @@
 #include "thttp.h"
 #include "tjson.h"
 
-#define TELEMETRY_SERVER "telemetry.taosdata.com"
-#define TELEMETRY_PORT   80
-
 typedef struct {
   int64_t numOfDnode;
   int64_t numOfMnode;
@@ -62,6 +59,12 @@ static void mndGetStat(SMnode* pMnode, SMnodeStat* pStat) {
 
     sdbRelease(pSdb, pVgroup);
   }
+
+  pStat->numOfChildTable = 100;
+  pStat->numOfColumn = 200;
+  pStat->totalPoints = 300;
+  pStat->totalStorage = 400;
+  pStat->compStorage = 500;
 }
 
 static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) {
@@ -122,12 +125,14 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) {
 static int32_t mndProcessTelemTimer(SNodeMsg* pReq) {
   SMnode*     pMnode = pReq->pNode;
   STelemMgmt* pMgmt = &pMnode->telemMgmt;
-  if (!pMgmt->enable) return 0;
+  if (!tsEnableTelem) return 0;
 
   taosWLockLatch(&pMgmt->lock);
   char* pCont = mndBuildTelemetryReport(pMnode);
   if (pCont != NULL) {
-    taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont), HTTP_FLAT);
+    if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) {
+      mError("failed to send telemetry msg");
+    }
     taosMemoryFree(pCont);
   }
   taosWUnLockLatch(&pMgmt->lock);
@@ -138,7 +143,6 @@ int32_t mndInitTelem(SMnode* pMnode) {
   STelemMgmt* pMgmt = &pMnode->telemMgmt;
 
   taosInitRWLatch(&pMgmt->lock);
-  pMgmt->enable = tsEnableTelemetryReporting;
   taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
   mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
 

source/dnode/mnode/impl/src/mndTopic.c

@@ -76,7 +76,13 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER);
   SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER);
   SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER);
+  SDB_SET_INT64(pRaw, dataPos, pTopic->subDbUid, TOPIC_ENCODE_OVER);
   SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER);
+  SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER);
+  SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER);
+  SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER);
+  SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER);
+  SDB_SET_INT8(pRaw, dataPos, pTopic->withTagSchema, TOPIC_ENCODE_OVER);
   SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
   SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER);
   SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER);
@@ -134,7 +140,13 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER);
   SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER);
   SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
+  SDB_GET_INT64(pRaw, dataPos, &pTopic->subDbUid, TOPIC_DECODE_OVER);
   SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
+  SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER);
+  SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER);
+  SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER);
+  SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER);
+  SDB_GET_INT8(pRaw, dataPos, &pTopic->withTagSchema, TOPIC_DECODE_OVER);
 
   SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
   pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char));
@@ -254,34 +266,14 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
     terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
     return -1;
   }
+
+  if ((pCreate->ast == NULL || pCreate->ast[0] == 0) && pCreate->subscribeDbName[0] == 0) {
+    terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
+    return -1;
+  }
   return 0;
 }
 
-#if 0
-static int32_t mndGetPlanString(const SCMCreateTopicReq *pCreate, char **pStr) {
-  if (NULL == pCreate->ast) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SNode  *pAst = NULL;
-  int32_t code = nodesStringToNode(pCreate->ast, &pAst);
-
-  SQueryPlan *pPlan = NULL;
-  if (TSDB_CODE_SUCCESS == code) {
-    SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
-    code = qCreateQueryPlan(&cxt, &pPlan, NULL);
-  }
-
-  if (TSDB_CODE_SUCCESS == code) {
-    code = nodesNodeToString(pPlan, false, pStr, NULL);
-  }
-  nodesDestroyNode(pAst);
-  nodesDestroyNode(pPlan);
-  terrno = code;
-  return code;
-}
-#endif
-
 static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
   mDebug("topic:%s to create", pCreate->name);
   SMqTopicObj topicObj = {0};
@@ -297,28 +289,38 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
   topicObj.ast = strdup(pCreate->ast);
   topicObj.astLen = strlen(pCreate->ast) + 1;
 
-  SNode *pAst = NULL;
+  if (pCreate->ast && pCreate->ast[0]) {
-  if (nodesStringToNode(pCreate->ast, &pAst) != 0) {
+    topicObj.subType = TOPIC_SUB_TYPE__TABLE;
-    mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+    topicObj.withTbName = 0;
-    return -1;
+    topicObj.withSchema = 0;
-  }
 
-  SQueryPlan *pPlan = NULL;
+    SNode *pAst = NULL;
+    if (nodesStringToNode(pCreate->ast, &pAst) != 0) {
+      mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+      return -1;
+    }
 
-  SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
+    SQueryPlan *pPlan = NULL;
-  if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) {
-    mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
-    return -1;
-  }
 
-  if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) {
+    SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
-    mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+    if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) {
-    return -1;
+      mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
-  }
+      return -1;
+    }
 
-  if (nodesNodeToString(pPlan, false, &topicObj.physicalPlan, NULL) != 0) {
+    if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) {
-    mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+      mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
-    return -1;
+      return -1;
+    }
+
+    if (nodesNodeToString(pPlan, false, &topicObj.physicalPlan, NULL) != 0) {
+      mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+      return -1;
+    }
+  } else {
+    topicObj.subType = TOPIC_SUB_TYPE__DB;
+    topicObj.withTbName = 1;
+    topicObj.withSchema = 1;
   }
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg);

source/dnode/mnode/impl/src/mnode.c

@@ -22,12 +22,14 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndFunc.h"
+#include "mndGrant.h"
 #include "mndInfoSchema.h"
-#include "mndPerfSchema.h"
 #include "mndMnode.h"
 #include "mndOffset.h"
+#include "mndPerfSchema.h"
 #include "mndProfile.h"
 #include "mndQnode.h"
+#include "mndQuery.h"
 #include "mndShow.h"
 #include "mndSma.h"
 #include "mndSnode.h"
@@ -40,12 +42,9 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
-#include "mndQuery.h"
-#include "mndGrant.h"
 
 #define MQ_TIMER_MS    3000
 #define TRNAS_TIMER_MS 6000
-#define TELEM_TIMER_MS 86400000
 
 static void *mndBuildTimerMsg(int32_t *pContLen) {
   SMTimerReq timerReq = {0};
@@ -97,7 +96,7 @@ static void mndPullupTelem(void *param, void *tmrId) {
     tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
   }
 
-  taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer);
+  taosTmrReset(mndPullupTelem, tsTelemInterval * 1000, pMnode, pMnode->timer, &pMnode->telemTimer);
 }
 
 static int32_t mndInitTimer(SMnode *pMnode) {
@@ -117,7 +116,8 @@ static int32_t mndInitTimer(SMnode *pMnode) {
     return -1;
   }
 
-  if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) {
+  int32_t interval = tsTelemInterval < 10 ? tsTelemInterval : 10;
+  if (taosTmrReset(mndPullupTelem, interval * 1000, pMnode, pMnode->timer, &pMnode->telemTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }

source/dnode/mnode/impl/test/func/func.cpp

@@ -32,6 +32,7 @@ void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) {
   int32_t len = strlen(pCode);
   pReq->pCode = (char*)taosMemoryCalloc(1, len + 1);
   strcpy(pReq->pCode, pCode);
+  pReq->codeLen = len;
 }
 
 void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) {
@@ -60,21 +61,6 @@ TEST_F(MndTestFunc, 02_Create_Func) {
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME);
   }
 
-  {
-    SCreateFuncReq createReq = {0};
-    strcpy(createReq.name, "f1");
-    SetCode(&createReq, "code1");
-
-    int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
-    void*   pReq = rpcMallocCont(contLen);
-    tSerializeSCreateFuncReq(pReq, contLen, &createReq);
-    tFreeSCreateFuncReq(&createReq);
-
-    SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
-    ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT);
-  }
-
   {
     SCreateFuncReq createReq = {0};
     strcpy(createReq.name, "f1");
@@ -90,22 +76,6 @@ TEST_F(MndTestFunc, 02_Create_Func) {
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE);
   }
 
-  {
-    SCreateFuncReq createReq = {0};
-    strcpy(createReq.name, "f1");
-    SetCode(&createReq, "code1");
-    SetComment(&createReq, "");
-
-    int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
-    void*   pReq = rpcMallocCont(contLen);
-    tSerializeSCreateFuncReq(pReq, contLen, &createReq);
-    tFreeSCreateFuncReq(&createReq);
-
-    SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
-    ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT);
-  }
-
   {
     SCreateFuncReq createReq = {0};
     strcpy(createReq.name, "f1");
@@ -329,7 +299,6 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
     EXPECT_EQ(pFuncInfo->bufSize, 6);
     EXPECT_EQ(pFuncInfo->signature, 18);
     EXPECT_EQ(pFuncInfo->commentSize, strlen("comment2") + 1);
-    EXPECT_EQ(pFuncInfo->codeSize, strlen("code2") + 1);
 
     EXPECT_STREQ("comment2", pFuncInfo->pComment);
     EXPECT_STREQ("code2", pFuncInfo->pCode);
@@ -368,7 +337,6 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
       EXPECT_EQ(pFuncInfo->bufSize, 6);
       EXPECT_EQ(pFuncInfo->signature, 18);
       EXPECT_EQ(pFuncInfo->commentSize, strlen("comment2") + 1);
-      EXPECT_EQ(pFuncInfo->codeSize, strlen("code2") + 1);
       EXPECT_STREQ("comment2", pFuncInfo->pComment);
       EXPECT_STREQ("code2", pFuncInfo->pCode);
     }

source/dnode/mnode/impl/test/profile/profile.cpp

@@ -30,8 +30,15 @@ int32_t  MndTestProfile::connId;
 TEST_F(MndTestProfile, 01_ConnectMsg) {
   SConnectReq connectReq = {0};
   connectReq.pid = 1234;
+
+  char passwd[] = "taosdata";
+  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
+  taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
+
   strcpy(connectReq.app, "mnode_test_profile");
   strcpy(connectReq.db, "");
+  strcpy(connectReq.user, "root");
+  strcpy(connectReq.passwd, secretEncrypt);
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = rpcMallocCont(contLen);
@@ -58,10 +65,16 @@ TEST_F(MndTestProfile, 01_ConnectMsg) {
 }
 
 TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) {
+  char passwd[] = "taosdata";
+  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
+  taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
+
   SConnectReq connectReq = {0};
   connectReq.pid = 1234;
   strcpy(connectReq.app, "mnode_test_profile");
   strcpy(connectReq.db, "invalid_db");
+  strcpy(connectReq.user, "root");
+  strcpy(connectReq.passwd, secretEncrypt);
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = rpcMallocCont(contLen);

source/dnode/mnode/impl/test/show/show.cpp

@@ -54,10 +54,16 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
 }
 
 TEST_F(MndTestShow, 03_ShowMsg_Conn) {
+  char passwd[] = "taosdata";
+  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
+  taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
+
   SConnectReq connectReq = {0};
   connectReq.pid = 1234;
   strcpy(connectReq.app, "mnode_test_show");
   strcpy(connectReq.db, "");
+  strcpy(connectReq.user, "root");
+  strcpy(connectReq.passwd, secretEncrypt);
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = rpcMallocCont(contLen);
@@ -74,4 +80,4 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) {
 TEST_F(MndTestShow, 04_ShowMsg_Cluster) {
   test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", "");
   EXPECT_EQ(test.GetShowRows(), 1);
-}
+}

source/dnode/vnode/inc/vnode.h

@@ -109,8 +109,7 @@ int     tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList
 int     tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
 int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReadHandle *pHandle);
-int     tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo);
+int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows);
-SArray *tqRetrieveDataBlock(STqReadHandle *pHandle);
 
 // need to reposition
 

source/dnode/vnode/src/inc/tq.h

@@ -16,6 +16,13 @@
 #ifndef _TD_VNODE_TQ_H_
 #define _TD_VNODE_TQ_H_
 
+#include "executor.h"
+#include "os.h"
+#include "thash.h"
+#include "tmsg.h"
+#include "ttimer.h"
+#include "wal.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -30,12 +37,6 @@ extern "C" {
 #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
-enum {
-  TQ_STREAM_TOKEN__DATA = 1,
-  TQ_STREAM_TOKEN__WATERMARK,
-  TQ_STREAM_TOKEN__CHECKPOINT,
-};
-
 #define TQ_BUFFER_SIZE 4
 
 #define TQ_BUCKET_MASK 0xFF
@@ -151,22 +152,27 @@ typedef struct {
 } STqMetaStore;
 
 typedef struct {
-  SMemAllocatorFactory* pAllocatorFactory;
+  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
-  SMemAllocator*        pAllocator;
+  int64_t consumerId;
-} STqMemRef;
+  int32_t epoch;
+  char*   qmsg;
+  // SRWLatch        lock;
+  SWalReadHandle* pReadHandle;
+  // number should be identical to fetch thread num
+  qTaskInfo_t task[4];
+} STqExec;
 
 struct STQ {
   // the collection of groups
   // the handle of meta kvstore
   bool          writeTrigger;
   char*         path;
-  STqMemRef     tqMemRef;
   STqMetaStore* tqMeta;
-  // STqPushMgr*   tqPushMgr;
+  SHashObj*     tqMetaNew;  // subKey -> tqExec
-  SHashObj* pStreamTasks;
+  SHashObj*     pStreamTasks;
-  SVnode*   pVnode;
+  SVnode*       pVnode;
-  SWal*     pWal;
+  SWal*         pWal;
-  SMeta*    pVnodeMeta;
+  SMeta*        pVnodeMeta;
 };
 
 typedef struct {
@@ -230,10 +236,6 @@ typedef struct {
   // TODO sync function
 } STqStreamPusher;
 
-typedef struct {
-  int8_t type;  // mq or stream
-} STqPusher;
-
 typedef struct {
   SHashObj* pHash;  // <id, STqPush*>
 } STqPushMgr;
@@ -257,9 +259,10 @@ int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
 int tqCommit(STQ*);
 
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
-int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessRebReq(STQ* pTq, char* msg);
+// int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
-int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
+// int32_t tqProcessRebReq(STQ* pTq, char* msg);
+// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
@@ -314,4 +317,4 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet
 }
 #endif
 
-#endif /*_TD_VNODE_TQ_H_*/
+#endif /*_TD_VNODE_TQ_H_*/

source/dnode/vnode/src/tq/tq.c

@@ -29,31 +29,14 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMe
   pTq->pVnode = pVnode;
   pTq->pWal = pWal;
   pTq->pVnodeMeta = pVnodeMeta;
-#if 0
-  pTq->tqMemRef.pAllocatorFactory = allocFac;
-  pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
-  if (pTq->tqMemRef.pAllocator == NULL) {
-    // TODO: error code of buffer pool
-  }
-#endif
   pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer,
                             (FTqDelete)taosMemoryFree, 0);
   if (pTq->tqMeta == NULL) {
     taosMemoryFree(pTq);
-#if 0
-    allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator);
-#endif
     return NULL;
   }
 
-#if 0
+  pTq->tqMetaNew = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
-  pTq->tqPushMgr = tqPushMgrOpen();
-  if (pTq->tqPushMgr == NULL) {
-    // free store
-    taosMemoryFree(pTq);
-    return NULL;
-  }
-#endif
 
   pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
 
@@ -248,16 +231,15 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 
   return 0;
 }
-#if 0
+
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
-  SMqPollReq* pReq = pMsg->pCont;
+  SMqPollReqV2* pReq = pMsg->pCont;
-  int64_t     consumerId = pReq->consumerId;
+  int64_t       consumerId = pReq->consumerId;
-  int64_t     fetchOffset;
+  int32_t       reqEpoch = pReq->epoch;
-  int64_t     blockingTime = pReq->blockingTime;
+  int64_t       fetchOffset;
-  int32_t     reqEpoch = pReq->epoch;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-    fetchOffset = 0;
+    fetchOffset = walGetFirstVer(pTq->pWal);
   } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
     fetchOffset = walGetLastVer(pTq->pWal);
   } else {
@@ -267,65 +249,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
          TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
-  SMqPollRsp rsp = {
+  STqExec* pExec = taosHashGet(pTq->tqMetaNew, pReq->subKey, strlen(pReq->subKey));
-      /*.consumerId = consumerId,*/
+  ASSERT(pExec);
-      .numOfTopics = 0,
-      .pBlockData = NULL,
-  };
 
-  STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
+  int32_t consumerEpoch = atomic_load_32(&pExec->epoch);
-  if (pConsumer == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch);
   while (consumerEpoch < reqEpoch) {
-    consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch);
+    consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch);
   }
 
-  STqTopic* pTopic = NULL;
+  SMqDataBlkRsp rsp = {0};
-  int32_t   sz = taosArrayGetSize(pConsumer->topics);
-  for (int32_t i = 0; i < sz; i++) {
-    STqTopic* topic = taosArrayGet(pConsumer->topics, i);
-    // TODO race condition
-    ASSERT(pConsumer->consumerId == consumerId);
-    if (strcmp(topic->topicName, pReq->topic) == 0) {
-      pTopic = topic;
-      break;
-    }
-  }
-  if (pTopic == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
-          TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
-         TD_VID(pTq->pVnode));
-
   rsp.reqOffset = pReq->currentOffset;
-  rsp.skipLogNum = 0;
+  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
+  rsp.blockData = taosArrayInit(0, sizeof(void*));
 
   while (1) {
-    /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
+    consumerEpoch = atomic_load_32(&pExec->epoch);
-    // TODO
-    consumerEpoch = atomic_load_32(&pConsumer->epoch);
     if (consumerEpoch > reqEpoch) {
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
              consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
+
     SWalReadHead* pHead;
-    if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
+    if (walReadWithHandle_s(pExec->pReadHandle, fetchOffset, &pHead) < 0) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
@@ -333,101 +279,88 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
              TD_VID(pTq->pVnode), fetchOffset);
       break;
     }
+
     vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
            TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
-    /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
+
-    /*pHead = pTopic->pReadhandle->pHead;*/
     if (pHead->msgType == TDMT_VND_SUBMIT) {
       SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
-      qTaskInfo_t task = pTopic->buffer.output[workerId].task;
+      qTaskInfo_t task = pExec->task[workerId];
       ASSERT(task);
       qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
-      SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
       while (1) {
         SSDataBlock* pDataBlock = NULL;
-        uint64_t     ts;
+        uint64_t     ts = 0;
         if (qExecTask(task, &pDataBlock, &ts) < 0) {
-          ASSERT(false);
+          ASSERT(0);
-        }
-        if (pDataBlock == NULL) {
-          /*pos = fetchOffset % TQ_BUFFER_SIZE;*/
-          break;
         }
+        if (pDataBlock == NULL) break;
 
-        taosArrayPush(pRes, pDataBlock);
+        ASSERT(pDataBlock->info.rows != 0);
+        ASSERT(pDataBlock->info.numOfCols != 0);
+
+        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
+        void*              buf = taosMemoryCalloc(1, dataStrLen);
+        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+        pRetrieve->useconds = ts;
+        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+        pRetrieve->compressed = 0;
+        pRetrieve->completed = 1;
+        pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
+
+        // TODO enable compress
+        int32_t actualLen = 0;
+        blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
+        actualLen += sizeof(SRetrieveTableRsp);
+        ASSERT(actualLen <= dataStrLen);
+        taosArrayPush(rsp.blockDataLen, &actualLen);
+        taosArrayPush(rsp.blockData, &buf);
+        rsp.blockNum++;
       }
-
-      if (taosArrayGetSize(pRes) == 0) {
-        vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
-               pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
-        fetchOffset++;
-        rsp.skipLogNum++;
-        taosArrayDestroy(pRes);
-        continue;
-      }
-      rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper;
-      rsp.rspOffset = fetchOffset;
-
-      rsp.numOfTopics = 1;
-      rsp.pBlockData = pRes;
-
-      int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
-      void*   buf = rpcMallocCont(tlen);
-      if (buf == NULL) {
-        pMsg->code = -1;
-        taosMemoryFree(pHead);
-        return -1;
-      }
-      ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
-      ((SMqRspHead*)buf)->epoch = pReq->epoch;
-      ((SMqRspHead*)buf)->consumerId = consumerId;
-
-      void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-      tEncodeSMqPollRsp(&abuf, &rsp);
-      /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/
-      pMsg->pCont = buf;
-      pMsg->contLen = tlen;
-      pMsg->code = 0;
-      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
-             pHead->msgType, consumerId, pReq->epoch);
-      tmsgSendRsp(pMsg);
-      taosMemoryFree(pHead);
-      return 0;
-    } else {
-      taosMemoryFree(pHead);
-      fetchOffset++;
-      rsp.skipLogNum++;
     }
+
+    // TODO batch optimization
+    if (rsp.blockNum != 0) break;
+    rsp.skipLogNum++;
+    fetchOffset++;
   }
 
-  /*if (blockingTime != 0) {*/
+  ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
-  /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
+  ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
-  /*} else {*/
+
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
+  if (rsp.blockNum != 0)
+    rsp.rspOffset = fetchOffset;
+  else
+    rsp.rspOffset = fetchOffset - 1;
+
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     pMsg->code = -1;
     return -1;
   }
+
   ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
   ((SMqRspHead*)buf)->epoch = pReq->epoch;
-  rsp.rspOffset = fetchOffset - 1;
+  ((SMqRspHead*)buf)->consumerId = consumerId;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqPollRsp(&abuf, &rsp);
+  tEncodeSMqDataBlkRsp(&abuf, &rsp);
-  rsp.pBlockData = NULL;
   pMsg->pCont = buf;
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
-         pReq->epoch);
-  /*}*/
 
+  vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+         TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
+
+  // TODO destroy
+  taosArrayDestroy(rsp.blockData);
+  taosArrayDestroy(rsp.blockDataLen);
   return 0;
 }
-#endif
 
+#if 0
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
@@ -436,7 +369,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   int32_t     reqEpoch = pReq->epoch;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-    fetchOffset = 0;
+    fetchOffset = walGetFirstVer(pTq->pWal);
   } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
     fetchOffset = walGetLastVer(pTq->pWal);
   } else {
@@ -635,124 +568,54 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   return 0;
 }
+#endif
 
-int32_t tqProcessRebReq(STQ* pTq, char* msg) {
+// TODO: persist meta into tdb
-  SMqMVRebReq req = {0};
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
-  terrno = TSDB_CODE_SUCCESS;
+  SMqRebVgReq req;
-  tDecodeSMqMVRebReq(msg, &req);
+  tDecodeSMqRebVgReq(msg, &req);
+  // todo lock
+  STqExec* pExec = taosHashGet(pTq->tqMetaNew, req.subKey, strlen(req.subKey));
+  if (pExec == NULL) {
+    ASSERT(req.oldConsumerId == -1);
+    ASSERT(req.newConsumerId != -1);
+    STqExec exec = {0};
+    pExec = &exec;
+    /*taosInitRWLatch(&pExec->lock);*/
 
-  vDebug("vg %d set from consumer %ld to consumer %ld", req.vgId, req.oldConsumerId, req.newConsumerId);
+    memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
-  STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId);
+    pExec->consumerId = req.newConsumerId;
-  ASSERT(pConsumer);
+    pExec->epoch = -1;
-  ASSERT(pConsumer->consumerId == req.oldConsumerId);
+    pExec->qmsg = req.qmsg;
-  int32_t numOfTopics = taosArrayGetSize(pConsumer->topics);
+    req.qmsg = NULL;
-  if (numOfTopics == 1) {
+    pExec->pReadHandle = walOpenReadHandle(pTq->pVnode->pWal);
-    STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0);
+    for (int32_t i = 0; i < 4; i++) {
-    ASSERT(strcmp(pTopic->topicName, req.topic) == 0);
+      STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
-    STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId);
+      SReadHandle    handle = {
-    if (pNewConsumer == NULL) {
+             .reader = pReadHandle,
-      pConsumer->consumerId = req.newConsumerId;
+             .meta = pTq->pVnodeMeta,
-      tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
+      };
-      tqHandleCommit(pTq->tqMeta, req.newConsumerId);
+      pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);
-      tqHandlePurge(pTq->tqMeta, req.oldConsumerId);
+      ASSERT(pExec->task[i]);
-      return 0;
-    } else {
-      taosArrayPush(pNewConsumer->topics, pTopic);
     }
+    taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
+    return 0;
   } else {
-    for (int32_t i = 0; i < numOfTopics; i++) {
+    /*if (req.newConsumerId != -1) {*/
-      STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
+    /*taosWLockLatch(&pExec->lock);*/
-      if (strcmp(pTopic->topicName, req.topic) == 0) {
+    ASSERT(pExec->consumerId == req.oldConsumerId);
-        STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId);
+    // TODO handle qmsg and exec modification
-        if (pNewConsumer == NULL) {
+    atomic_store_32(&pExec->epoch, -1);
-          pNewConsumer = taosMemoryCalloc(1, sizeof(STqConsumer));
+    atomic_store_64(&pExec->consumerId, req.newConsumerId);
-          if (pNewConsumer == NULL) {
+    atomic_add_fetch_32(&pExec->epoch, 1);
-            terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
+    /*taosWUnLockLatch(&pExec->lock);*/
-            return -1;
+    return 0;
-          }
+    /*} else {*/
-          strcpy(pNewConsumer->cgroup, pConsumer->cgroup);
+    // TODO
-          pNewConsumer->topics = taosArrayInit(0, sizeof(STqTopic));
+    /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/
-          pNewConsumer->consumerId = req.newConsumerId;
+    /*return 0;*/
-          pNewConsumer->epoch = 0;
+    /*}*/
-
-          taosArrayPush(pNewConsumer->topics, pTopic);
-          tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
-          tqHandleCommit(pTq->tqMeta, req.newConsumerId);
-          return 0;
-        }
-        ASSERT(pNewConsumer->consumerId == req.newConsumerId);
-        taosArrayPush(pNewConsumer->topics, pTopic);
-        break;
-      }
-    }
-    //
   }
-  return 0;
-}
-
-int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
-  SMqSetCVgReq req = {0};
-  tDecodeSMqSetCVgReq(msg, &req);
-  bool create = false;
-
-  vDebug("vg %d set to consumer %ld", req.vgId, req.consumerId);
-  STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.consumerId);
-  if (pConsumer == NULL) {
-    pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer));
-    if (pConsumer == NULL) {
-      terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-      return -1;
-    }
-    strcpy(pConsumer->cgroup, req.cgroup);
-    pConsumer->topics = taosArrayInit(0, sizeof(STqTopic));
-    pConsumer->consumerId = req.consumerId;
-    pConsumer->epoch = 0;
-    create = true;
-  }
-
-  STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic));
-  if (pTopic == NULL) {
-    taosArrayDestroy(pConsumer->topics);
-    taosMemoryFree(pConsumer);
-    return -1;
-  }
-  strcpy(pTopic->topicName, req.topicName);
-  pTopic->sql = req.sql;
-  pTopic->physicalPlan = req.physicalPlan;
-  pTopic->qmsg = req.qmsg;
-  /*pTopic->committedOffset = -1;*/
-  /*pTopic->currentOffset = -1;*/
-
-  pTopic->buffer.firstOffset = -1;
-  pTopic->buffer.lastOffset = -1;
-  pTopic->pReadhandle = walOpenReadHandle(pTq->pWal);
-  if (pTopic->pReadhandle == NULL) {
-    ASSERT(false);
-  }
-  for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
-    pTopic->buffer.output[i].status = 0;
-    STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
-    SReadHandle    handle = {
-           .reader = pReadHandle,
-           .meta = pTq->pVnodeMeta,
-    };
-    pTopic->buffer.output[i].pReadHandle = pReadHandle;
-    pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
-    ASSERT(pTopic->buffer.output[i].task);
-  }
-  vDebug("set topic %s to consumer %ld on vg %d", pTopic->topicName, req.consumerId, TD_VID(pTq->pVnode));
-  taosArrayPush(pConsumer->topics, pTopic);
-  if (create) {
-    tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
-    tqHandleCommit(pTq->tqMeta, req.consumerId);
-  }
-  terrno = TSDB_CODE_SUCCESS;
-  return 0;
-}
-
-int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) {
-  terrno = TSDB_CODE_SUCCESS;
-  return 0;
 }
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #include "vnodeInt.h"
-// TODO:replace by an abstract file layer
 // #include <fcntl.h>
 // #include <string.h>
 // #include <unistd.h>

source/dnode/vnode/src/tq/tqRead.c

@@ -82,16 +82,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
   return false;
 }
 
-int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) {
+int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows) {
-  // currently only rows are used
-
-  pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList);
-  pBlockInfo->rows = pHandle->pBlock->numOfRows;
-  //  pBlockInfo->uid = pHandle->pBlock->uid; // the uid can not be assigned to pBlockData.
-  return 0;
-}
-
-SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
   /*int32_t         sversion = pHandle->pBlock->sversion;*/
   // TODO set to real sversion
   int32_t sversion = 0;
@@ -112,7 +103,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
   STSchema*       pTschema = pHandle->pSchema;
   SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
 
-  int32_t numOfRows = pHandle->pBlock->numOfRows;
+  *pNumOfRows = pHandle->pBlock->numOfRows;
   /*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
   int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
 
@@ -120,10 +111,11 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
     colNumNeed = pSchemaWrapper->nCols;
   }
 
-  SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData));
+  *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData));
-  if (pArray == NULL) {
+  if (*ppCols == NULL) {
-    return NULL;
+    return -1;
   }
+
   int32_t colMeta = 0;
   int32_t colNeed = 0;
   while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
@@ -136,21 +128,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       colNeed++;
     } else {
       SColumnInfoData colInfo = {0};
-      /*int             sz = numOfRows * pColSchema->bytes;*/
       colInfo.info.bytes = pColSchema->bytes;
       colInfo.info.colId = pColSchema->colId;
       colInfo.info.type = pColSchema->type;
 
-      if (colInfoDataEnsureCapacity(&colInfo, 0, numOfRows) < 0) {
+      if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
-        taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
+        goto FAIL;
-        return NULL;
       }
-      taosArrayPush(pArray, &colInfo);
+      taosArrayPush(*ppCols, &colInfo);
       colMeta++;
       colNeed++;
     }
   }
 
+  int32_t colActual = taosArrayGetSize(*ppCols);
+
+  // TODO in stream shuffle case, fetch groupId
+  *pGroupId = 0;
+
   STSRowIter iter = {0};
   tdSTSRowIterInit(&iter, pTschema);
   STSRow* row;
@@ -159,22 +154,22 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
     tdSTSRowIterReset(&iter, row);
     // get all wanted col of that block
-    int32_t colTot = taosArrayGetSize(pArray);
+    for (int32_t i = 0; i < colActual; i++) {
-    for (int32_t i = 0; i < colTot; i++) {
+      SColumnInfoData* pColData = taosArrayGet(*ppCols, i);
-      SColumnInfoData* pColData = taosArrayGet(pArray, i);
       SCellVal         sVal = {0};
       if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
         break;
       }
-      /*if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {*/
       if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
-        taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
+        goto FAIL;
-        return NULL;
       }
     }
     curRow++;
   }
-  return pArray;
+  return 0;
+FAIL:
+  taosArrayDestroy(*ppCols);
+  return -1;
 }
 
 void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1350,7 +1350,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
       if (tBitmaps > 0) {
         bptr = POINTER_SHIFT(pBlockData, lsize + flen);
         if (isSuper && !tdDataColsIsBitmapI(pDataCols)) {
-          tdMergeBitmap((uint8_t *)pDataCol->pBitmap, nBitmaps, (uint8_t *)pDataCol->pBitmap);
+          tdMergeBitmap((uint8_t *)pDataCol->pBitmap, rowsToWrite, (uint8_t *)pDataCol->pBitmap);
         }
         tBitmapsLen =
             tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES,

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -305,7 +305,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
       SDataCol *pDataCol = pReadh->pDCols[0]->cols + i;
       if (pDataCol->bitmap) {
         ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID);
-        tdMergeBitmap(pDataCol->pBitmap, TD_BITMAP_BYTES(pReadh->pDCols[0]->numOfRows), pDataCol->pBitmap);
+        tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap);
         tdDataColsSetBitmapI(pReadh->pDCols[0]);
       }
     }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -80,6 +80,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       pRsp->msgType = TDMT_VND_SUBMIT_RSP;
       vnodeProcessSubmitReq(pVnode, ptr, pRsp);
       break;
+    case TDMT_VND_MQ_VG_CHANGE:
+      if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
+        // TODO: handle error
+      }
+      break;
+#if 0
     case TDMT_VND_MQ_SET_CONN: {
       if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
         // TODO: handle error
@@ -93,6 +100,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
       }
     } break;
+#endif
     case TDMT_VND_TASK_DEPLOY: {
       if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
@@ -309,4 +317,4 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg
   pRsp->contLen = sizeof(SSubmitRsp);
 
   return 0;
-}
+}

source/libs/executor/src/scanoperator.c

@@ -30,12 +30,11 @@
 #include "query.h"
 #include "tcompare.h"
 #include "thash.h"
-#include "vnode.h"
 #include "ttypes.h"
+#include "vnode.h"
 
 #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
-#define SWITCH_ORDER(n)           (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
+#define SWITCH_ORDER(n)              (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
-
 
 void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   for (int32_t i = 0; i < numOfOutput; ++i) {
@@ -90,7 +89,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord
   }
 
   int64_t key = tw->skey, interval = pInterval->interval;
-  //convert key to second
+  // convert key to second
   key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000;
 
   if (pInterval->intervalUnit == 'y') {
@@ -98,7 +97,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord
   }
 
   struct tm tm;
-  time_t t = (time_t)key;
+  time_t    t = (time_t)key;
   taosLocalTime(&t, &tm);
 
   int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
@@ -125,8 +124,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
   // todo handle the time range case
   TSKEY sk = INT64_MIN;
   TSKEY ek = INT64_MAX;
-//  TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
+  //  TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
-//  TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
+  //  TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
 
   if (true) {
     getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey, &w);
@@ -136,7 +135,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
       return true;
     }
 
-    while(1) { // todo handle the desc order scan case
+    while (1) {  // todo handle the desc order scan case
       getNextTimeWindow(pInterval, &w, TSDB_ORDER_ASC);
       if (w.skey > pBlockInfo->window.ekey) {
         break;
@@ -148,31 +147,31 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
       }
     }
   } else {
-//    getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w);
+    //    getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w);
-//    assert(w.skey <= pBlockInfo->window.ekey);
+    //    assert(w.skey <= pBlockInfo->window.ekey);
-//
+    //
-//    if (w.skey > pBlockInfo->window.skey) {
+    //    if (w.skey > pBlockInfo->window.skey) {
-//      return true;
+    //      return true;
-//    }
+    //    }
-//
+    //
-//    while(1) {
+    //    while(1) {
-//      getNextTimeWindow(pQueryAttr, &w);
+    //      getNextTimeWindow(pQueryAttr, &w);
-//      if (w.ekey < pBlockInfo->window.skey) {
+    //      if (w.ekey < pBlockInfo->window.skey) {
-//        break;
+    //        break;
-//      }
+    //      }
-//
+    //
-//      assert(w.skey < pBlockInfo->window.skey);
+    //      assert(w.skey < pBlockInfo->window.skey);
-//      if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
+    //      if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
-//        return true;
+    //        return true;
-//      }
+    //      }
-//    }
+    //    }
   }
 
   return false;
 }
 
 int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) {
-  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
   STableScanInfo* pInfo = pOperator->info;
 
   STaskCostInfo* pCost = &pTaskInfo->cost;
@@ -189,13 +188,13 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
   taosMemoryFreeClear(pBlock->pBlockAgg);
 
   if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
-    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
+    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.ekey, pBlockInfo->rows);
+           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
     pCost->filterOutBlocks += 1;
     return TSDB_CODE_SUCCESS;
   } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
-    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
+    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.ekey, pBlockInfo->rows);
+           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
     pCost->skipBlocks += 1;
     return TSDB_CODE_SUCCESS;
   } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
@@ -218,12 +217,12 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
       }
 
       return TSDB_CODE_SUCCESS;
-    } else { // failed to load the block sma data, data block statistics does not exist, load data block instead
+    } else {  // failed to load the block sma data, data block statistics does not exist, load data block instead
       *status = FUNC_DATA_REQUIRED_DATA_LOAD;
     }
   }
 
-  ASSERT (*status == FUNC_DATA_REQUIRED_DATA_LOAD);
+  ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
 
   // todo filter data block according to the block sma data firstly
 #if 0
@@ -249,8 +248,8 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
   doFilter(pTableScanInfo->pFilterNode, pBlock);
   if (pBlock->info.rows == 0) {
     pCost->filterOutBlocks += 1;
-    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
+    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.ekey, pBlockInfo->rows);
+           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -348,9 +347,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
     setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
     pTableScanInfo->scanFlag = REPEAT_SCAN;
 
-//    if (pResultRowInfo->size > 0) {
+    //    if (pResultRowInfo->size > 0) {
-//      pResultRowInfo->curPos = 0;
+    //      pResultRowInfo->curPos = 0;
-//    }
+    //    }
 
     qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
            GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
@@ -367,7 +366,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
            GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
 
     if (pResultRowInfo->size > 0) {
-//      pResultRowInfo->curPos = pResultRowInfo->size - 1;
+      //      pResultRowInfo->curPos = pResultRowInfo->size - 1;
     }
 
     p = doTableScanImpl(pOperator, newgroup);
@@ -376,9 +375,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
   return p;
 }
 
-SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag,
+SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput,
-                                           int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock,
+                                           int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime,
-                                           SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
+                                           SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition,
+                                           SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
   assert(repeatTime > 0);
 
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
@@ -391,26 +391,26 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
     return NULL;
   }
 
-  pInfo->interval         = *pInterval;
+  pInfo->interval = *pInterval;
-  pInfo->sampleRatio      = sampleRatio;
+  pInfo->sampleRatio = sampleRatio;
-  pInfo->dataBlockLoadFlag= dataLoadFlag;
+  pInfo->dataBlockLoadFlag = dataLoadFlag;
-  pInfo->pResBlock        = pResBlock;
+  pInfo->pResBlock = pResBlock;
-  pInfo->pFilterNode      = pCondition;
+  pInfo->pFilterNode = pCondition;
-  pInfo->dataReader       = pTsdbReadHandle;
+  pInfo->dataReader = pTsdbReadHandle;
-  pInfo->times            = repeatTime;
+  pInfo->times = repeatTime;
-  pInfo->reverseTimes     = reverseTime;
+  pInfo->reverseTimes = reverseTime;
-  pInfo->order            = order;
+  pInfo->order = order;
-  pInfo->current          = 0;
+  pInfo->current = 0;
-  pInfo->scanFlag         = MAIN_SCAN;
+  pInfo->scanFlag = MAIN_SCAN;
-  pInfo->pColMatchInfo    = pColMatchInfo;
+  pInfo->pColMatchInfo = pColMatchInfo;
-  pOperator->name         = "TableScanOperator";
+  pOperator->name = "TableScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
   pOperator->blockingOptr = false;
-  pOperator->status       = OP_NOT_OPENED;
+  pOperator->status = OP_NOT_OPENED;
-  pOperator->info         = pInfo;
+  pOperator->info = pInfo;
-  pOperator->numOfOutput  = numOfOutput;
+  pOperator->numOfOutput = numOfOutput;
-  pOperator->getNextFn    = doTableScan;
+  pOperator->getNextFn = doTableScan;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   static int32_t cost = 0;
   pOperator->cost.openCost = ++cost;
@@ -456,67 +456,67 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) {
     ++numRowSteps;
   }
 
-  tableBlockDist.dataBlockInfos  = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));
+  tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));
   taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps);
 
   tableBlockDist.maxRows = INT_MIN;
   tableBlockDist.minRows = INT_MAX;
 
   tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist);
-  tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader);
+  tableBlockDist.numOfRowsInMemTable = (int32_t)tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader);
 
   SSDataBlock* pBlock = pTableScanInfo->pResBlock;
-  pBlock->info.rows   = 1;
+  pBlock->info.rows = 1;
   pBlock->info.numOfCols = 1;
 
-//  SBufferWriter bw = tbufInitWriter(NULL, false);
+  //  SBufferWriter bw = tbufInitWriter(NULL, false);
-//  blockDistInfoToBinary(&tableBlockDist, &bw);
+  //  blockDistInfoToBinary(&tableBlockDist, &bw);
   SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
 
-//  int32_t len = (int32_t) tbufTell(&bw);
+  //  int32_t len = (int32_t) tbufTell(&bw);
-//  pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
+  //  pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
-//  *(int32_t*) pColInfo->pData = len;
+  //  *(int32_t*) pColInfo->pData = len;
-//  memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
+  //  memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
-//
+  //
-//  tbufCloseWriter(&bw);
+  //  tbufCloseWriter(&bw);
 
-//  SArray* g = GET_TABLEGROUP(pOperator->, 0);
+  //  SArray* g = GET_TABLEGROUP(pOperator->, 0);
-//  pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
+  //  pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
 
   pOperator->status = OP_EXEC_DONE;
   return pBlock;
 }
 
 SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) {
-  STableScanInfo* pInfo    = taosMemoryCalloc(1, sizeof(STableScanInfo));
+  STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
-  SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
     pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
     goto _error;
   }
 
-  pInfo->dataReader       = dataReader;
+  pInfo->dataReader = dataReader;
-//  pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+  //  pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
 
   SColumnInfoData infoData = {0};
-  infoData.info.type  = TSDB_DATA_TYPE_BINARY;
+  infoData.info.type = TSDB_DATA_TYPE_BINARY;
   infoData.info.bytes = 1024;
   infoData.info.colId = 0;
-//  taosArrayPush(pInfo->block.pDataBlock, &infoData);
+  //  taosArrayPush(pInfo->block.pDataBlock, &infoData);
 
-  pOperator->name          = "DataBlockInfoScanOperator";
+  pOperator->name = "DataBlockInfoScanOperator";
   //  pOperator->operatorType = OP_TableBlockInfoScan;
-  pOperator->blockingOptr  = false;
+  pOperator->blockingOptr = false;
-  pOperator->status        = OP_NOT_OPENED;
+  pOperator->status = OP_NOT_OPENED;
-  pOperator->_openFn       = operatorDummyOpenFn;
+  pOperator->_openFn = operatorDummyOpenFn;
-  pOperator->getNextFn     = doBlockInfoScan;
+  pOperator->getNextFn = doBlockInfoScan;
 
-  pOperator->info          = pInfo;
+  pOperator->info = pInfo;
-  pOperator->pTaskInfo     = pTaskInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   return pOperator;
 
-  _error:
+_error:
   taosMemoryFreeClear(pInfo);
   taosMemoryFreeClear(pOperator);
   return NULL;
@@ -558,29 +558,42 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
     blockDataCleanup(pInfo->pRes);
 
     while (tqNextDataBlock(pInfo->readerHandle)) {
-      pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo);
+      SArray*  pCols = NULL;
-      if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
+      uint64_t groupId;
-        terrno = pTaskInfo->code;
+      int32_t  numOfRows;
-        pOperator->status = OP_EXEC_DONE;
+      int32_t  code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows);
+
+      if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
+        pTaskInfo->code = code;
         return NULL;
       }
 
-      if (pBlockInfo->rows == 0) {
+      pInfo->pRes->info.groupId = groupId;
-        break;
+      pInfo->pRes->info.rows = numOfRows;
-      }
-
-      SArray* pCols = tqRetrieveDataBlock(pInfo->readerHandle);
 
       int32_t numOfCols = pInfo->pRes->info.numOfCols;
       for (int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData* p = taosArrayGet(pCols, i);
+        SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
-        SColMatchInfo*   pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
         if (!pColMatchInfo->output) {
           continue;
         }
 
-        ASSERT(pColMatchInfo->colId == p->info.colId);
+        bool colExists = false;
-        taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, p);
+        for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) {
+          SColumnInfoData* pResCol = taosArrayGet(pCols, j);
+          if (pResCol->info.colId == pColMatchInfo->colId) {
+            taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol);
+            colExists = true;
+            break;
+          }
+        }
+
+        // the required column does not exists in submit block, let's set it to be all null value
+        if (!colExists) {
+          SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
+          colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows);
+          colDataAppendNNULL(pDst, 0, pBlockInfo->rows);
+        }
       }
 
       if (pInfo->pRes->pDataBlock == NULL) {
@@ -605,7 +618,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
   }
 }
 
-SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList,
+                                            SArray* pTableIdList, SExecTaskInfo* pTaskInfo) {
   SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
   SOperatorInfo*        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -618,7 +632,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
   int32_t numOfOutput = taosArrayGetSize(pColList);
 
   SArray* pColIds = taosArrayInit(4, sizeof(int16_t));
-  for(int32_t i = 0; i < numOfOutput; ++i) {
+  for (int32_t i = 0; i < numOfOutput; ++i) {
     int16_t* id = taosArrayGet(pColList, i);
     taosArrayPush(pColIds, id);
   }
@@ -644,16 +658,16 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
   pInfo->readerHandle = streamReadHandle;
   pInfo->pRes = pResBlock;
 
-  pOperator->name         = "StreamBlockScanOperator";
+  pOperator->name = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
   pOperator->blockingOptr = false;
-  pOperator->status       = OP_NOT_OPENED;
+  pOperator->status = OP_NOT_OPENED;
-  pOperator->info         = pInfo;
+  pOperator->info = pInfo;
-  pOperator->numOfOutput  = pResBlock->info.numOfCols;
+  pOperator->numOfOutput = pResBlock->info.numOfCols;
-  pOperator->_openFn      = operatorDummyOpenFn;
+  pOperator->_openFn = operatorDummyOpenFn;
-  pOperator->getNextFn    = doStreamBlockScan;
+  pOperator->getNextFn = doStreamBlockScan;
-  pOperator->closeFn      = operatorDummyCloseFn;
+  pOperator->closeFn = operatorDummyCloseFn;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   return pOperator;
 }
@@ -733,9 +747,9 @@ static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t
 
     SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp;
     pRsp->numOfRows = htonl(pRsp->numOfRows);
-    pRsp->useconds  = htobe64(pRsp->useconds);
+    pRsp->useconds = htobe64(pRsp->useconds);
-    pRsp->handle    = htobe64(pRsp->handle);
+    pRsp->handle = htobe64(pRsp->handle);
-    pRsp->compLen   = htonl(pRsp->compLen);
+    pRsp->compLen = htonl(pRsp->compLen);
   } else {
     operator->pTaskInfo->code = code;
   }
@@ -777,7 +791,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
         if (rowRes[j] == 0) {
           continue;
         }
-      
+
         colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false);
         numOfRow += 1;
       }
@@ -828,7 +842,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
         SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i);
         int64_t          tmp = 0;
         char             t[10] = {0};
-        STR_TO_VARSTR(t, "_");  //TODO
+        STR_TO_VARSTR(t, "_");  // TODO
         if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
           colDataAppend(pColInfoData, numOfRows, t, false);
         } else {
@@ -926,12 +940,12 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
     return NULL;
   }
 
-  pInfo->accountId   = accountId;
+  pInfo->accountId = accountId;
   pInfo->showRewrite = showRewrite;
-  pInfo->pRes        = pResBlock;
+  pInfo->pRes = pResBlock;
-  pInfo->capacity    = 4096;
+  pInfo->capacity = 4096;
-  pInfo->pCondition  = pCondition;
+  pInfo->pCondition = pCondition;
-  pInfo->scanCols    = colList;
+  pInfo->scanCols = colList;
 
   // TODO remove it
   int32_t     tableType = 0;
@@ -986,9 +1000,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
     tableType = TSDB_MGMT_TABLE_CONNS;
   } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, tListLen(pName->tname)) == 0) {
     tableType = TSDB_MGMT_TABLE_QUERIES;
-  }  else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) {
+  } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) {
     tableType = TSDB_MGMT_TABLE_VNODES;
-  }else {
+  } else {
     ASSERT(0);
   }
 
@@ -1025,15 +1039,15 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
 #endif
   }
 
-  pOperator->name         = "SysTableScanOperator";
+  pOperator->name = "SysTableScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN;
   pOperator->blockingOptr = false;
-  pOperator->status       = OP_NOT_OPENED;
+  pOperator->status = OP_NOT_OPENED;
-  pOperator->info         = pInfo;
+  pOperator->info = pInfo;
-  pOperator->numOfOutput  = pResBlock->info.numOfCols;
+  pOperator->numOfOutput = pResBlock->info.numOfCols;
-  pOperator->getNextFn    = doSysTableScan;
+  pOperator->getNextFn = doSysTableScan;
-  pOperator->closeFn      = destroySysScanOperator;
+  pOperator->closeFn = destroySysScanOperator;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   return pOperator;
 }

source/libs/function/CMakeLists.txt

@@ -14,7 +14,7 @@ target_include_directories(
 
 target_link_libraries(
     function
-    PRIVATE os util common nodes scalar
+    PRIVATE os util common nodes scalar catalog qcom transport
     PUBLIC uv_a
 )
 

source/libs/function/inc/builtins.h

@@ -20,26 +20,7 @@
 extern "C" {
 #endif
 
-#include "functionMgt.h"
+#include "functionMgtInt.h"
-
-#define FUNCTION_NAME_MAX_LENGTH 32
-
-#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n)    (1 << n)
-
-#define FUNC_MGT_AGG_FUNC               FUNC_MGT_FUNC_CLASSIFICATION_MASK(0)
-#define FUNC_MGT_SCALAR_FUNC            FUNC_MGT_FUNC_CLASSIFICATION_MASK(1)
-#define FUNC_MGT_NONSTANDARD_SQL_FUNC   FUNC_MGT_FUNC_CLASSIFICATION_MASK(2)
-#define FUNC_MGT_STRING_FUNC            FUNC_MGT_FUNC_CLASSIFICATION_MASK(3)
-#define FUNC_MGT_DATETIME_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
-#define FUNC_MGT_TIMELINE_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
-#define FUNC_MGT_TIMEORDER_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
-#define FUNC_MGT_PSEUDO_COLUMN_FUNC     FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
-#define FUNC_MGT_WINDOW_PC_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
-#define FUNC_MGT_SPECIAL_DATA_REQUIRED  FUNC_MGT_FUNC_CLASSIFICATION_MASK(9)
-#define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10)
-#define FUNC_MGT_MULTI_RES_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(11)
-
-#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 
 typedef int32_t (*FTranslateFunc)(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
 typedef EFuncDataRequired (*FFuncDataRequired)(SFunctionNode* pFunc, STimeWindow* pTimeWindow);

source/libs/function/inc/fnLog.h

@@ -0,0 +1,24 @@
+//
+// Created by slzhou on 22-4-20.
+//
+
+#ifndef TDENGINE_FNLOG_H
+#define TDENGINE_FNLOG_H
+#include "tlog.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define fnFatal(...) { if (fnDebugFlag & DEBUG_FATAL) { taosPrintLog("FN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
+#define fnError(...) { if (fnDebugFlag & DEBUG_ERROR) { taosPrintLog("FN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
+#define fnWarn(...)  { if (fnDebugFlag & DEBUG_WARN)  { taosPrintLog("FN WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
+#define fnInfo(...)  { if (fnDebugFlag & DEBUG_INFO)  { taosPrintLog("FN ", DEBUG_INFO, 255, __VA_ARGS__); }}
+#define fnDebug(...) { if (fnDebugFlag & DEBUG_DEBUG) { taosPrintLog("FN ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
+#define fnTrace(...) { if (fnDebugFlag & DEBUG_TRACE) { taosPrintLog("FN ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_FNLOG_H

source/libs/function/inc/functionMgtInt.h

@@ -21,9 +21,29 @@ extern "C" {
 #endif
 
 #include "functionMgt.h"
-// #include "builtins.h"
 
+#define FUNCTION_NAME_MAX_LENGTH 32
 
+#define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n)    (1 << n)
+
+#define FUNC_MGT_AGG_FUNC               FUNC_MGT_FUNC_CLASSIFICATION_MASK(0)
+#define FUNC_MGT_SCALAR_FUNC            FUNC_MGT_FUNC_CLASSIFICATION_MASK(1)
+#define FUNC_MGT_NONSTANDARD_SQL_FUNC   FUNC_MGT_FUNC_CLASSIFICATION_MASK(2)
+#define FUNC_MGT_STRING_FUNC            FUNC_MGT_FUNC_CLASSIFICATION_MASK(3)
+#define FUNC_MGT_DATETIME_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
+#define FUNC_MGT_TIMELINE_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
+#define FUNC_MGT_TIMEORDER_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
+#define FUNC_MGT_PSEUDO_COLUMN_FUNC     FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
+#define FUNC_MGT_WINDOW_PC_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
+#define FUNC_MGT_SPECIAL_DATA_REQUIRED  FUNC_MGT_FUNC_CLASSIFICATION_MASK(9)
+#define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10)
+#define FUNC_MGT_MULTI_RES_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(11)
+
+#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
+
+#define FUNC_UDF_ID_START_OFFSET_VAL 5000
+
+extern const int funcMgtUdfNum;
 
 #ifdef __cplusplus
 }

source/libs/function/inc/tudf.h

@@ -26,6 +26,9 @@
 extern "C" {
 #endif
 
+#define UDF_LISTEN_PIPE_NAME_LEN 32
+#define UDF_LISTEN_PIPE_NAME_PREFIX "udf.sock."
+
 //======================================================================================
 //begin API to taosd and qworker
 
@@ -35,17 +38,28 @@ enum {
   UDFC_CODE_PIPE_READ_ERR = -3,
 };
 
+/*TODO: no api for dnode startudfd/stopudfd*/
 /**
- * start udf dameon service
+ * start udfd dameon service
- * @return error code
  */
-int32_t startUdfService();
+int32_t startUdfd(int32_t dnodeId);
 
 /**
- * stop udf dameon service
+ * stop udfd dameon service
+ */
+int32_t stopUdfd(int32_t dnodeId);
+
+/**
+ * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
  * @return error code
  */
-int32_t stopUdfService();
+int32_t createUdfdProxy(int32_t dnodeId);
+
+/**
+ * destroy udfd proxy
+ * @return error code
+ */
+int32_t destroyUdfdProxy(int32_t dnodeId);
 
 typedef void *UdfHandle;
 
@@ -101,7 +115,6 @@ typedef struct SUdfInterBuf {
   char* buf;
 } SUdfInterBuf;
 
-//TODO: translate these calls to callUdf
 // output: interBuf
 int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf);
 // input: block, state

source/libs/function/inc/udfc.h

@@ -32,9 +32,9 @@ typedef struct SUdfInfo {
 
 typedef void *UdfHandle;
 
-int32_t startUdfService();
+int32_t createUdfdProxy();
 
-int32_t stopUdfService();
+int32_t destroyUdfdProxy();
 
 //int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles);
 

source/libs/function/src/builtins.c

@@ -32,7 +32,7 @@ static int32_t invaildFuncParaNumErrMsg(char* pErrBuf, int32_t len, const char*
 }
 
 static int32_t invaildFuncParaTypeErrMsg(char* pErrBuf, int32_t len, const char* pFuncName) {
-  return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid datatypes : %s", pFuncName);
+  return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid parameter data type : %s", pFuncName);
 }
 
 static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char* pFuncName) {
@@ -327,6 +327,10 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
       para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
+  if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
+      (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
   int32_t para2Bytes = pFunc->node.resType.bytes;
   if (para2Bytes <= 0) { //non-positive value or overflow
     return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
@@ -935,7 +939,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .translateFunc = translateToJson,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,
-    .sprocessFunc = NULL,
+    .sprocessFunc = toJsonFunction,
     .finalizeFunc = NULL
   }
 };

source/libs/function/src/functionMgt.c

@@ -20,16 +20,22 @@
 #include "taoserror.h"
 #include "thash.h"
 #include "builtins.h"
+#include "catalog.h"
 
 typedef struct SFuncMgtService {
   SHashObj* pFuncNameHashTable;
+  SArray* pUdfTable; // SUdfInfo
 } SFuncMgtService;
 
+typedef struct SUdfInfo {
+  SDataType outputDt;
+} SUdfInfo;
+
 static SFuncMgtService gFunMgtService;
 static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT;
 static int32_t initFunctionCode = 0;
 
-static void doInitFunctionHashTable() {
+static void doInitFunctionTable() {
   gFunMgtService.pFuncNameHashTable = taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   if (NULL == gFunMgtService.pFuncNameHashTable) {
     initFunctionCode = TSDB_CODE_FAILED;
@@ -42,6 +48,8 @@ static void doInitFunctionHashTable() {
       return;
     }
   }
+
+  gFunMgtService.pUdfTable = NULL;
 }
 
 static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
@@ -51,25 +59,56 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
   return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
 }
 
+static int32_t getUdfId(const char* pFuncName) {
+  // todo: udf by call catalog
+  if (1) {
+    return -1;
+  }
+  if (NULL == gFunMgtService.pUdfTable) {
+    gFunMgtService.pUdfTable = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SUdfInfo));
+  }
+  SUdfInfo info = {0}; //todo
+  taosArrayPush(gFunMgtService.pUdfTable, &info);
+  return taosArrayGetSize(gFunMgtService.pUdfTable) + FUNC_UDF_ID_START_OFFSET_VAL;
+}
+
+static int32_t getFuncId(const char* pFuncName) {
+  void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName));
+  if (NULL == pVal) {
+    return getUdfId(pFuncName);
+  }
+  return *(int32_t*)pVal;
+}
+
+static int32_t getUdfResultType(SFunctionNode* pFunc) {
+  SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, pFunc->funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1);
+  pFunc->node.resType = pUdf->outputDt;
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t fmFuncMgtInit() {
-  taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable);
+  taosThreadOnce(&functionHashTableInit, doInitFunctionTable);
   return initFunctionCode;
 }
 
 int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) {
-  void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName));
+  *pFuncId = getFuncId(pFuncName);
-  if (NULL == pVal) {
+  if (*pFuncId < 0) {
     return TSDB_CODE_FAILED;
   }
-  *pFuncId = *(int32_t*)pVal;
+  if (fmIsUserDefinedFunc(*pFuncId)) {
-  if (*pFuncId < 0 || *pFuncId >= funcMgtBuiltinsNum) {
+    *pFuncType = FUNCTION_TYPE_UDF;
-    return TSDB_CODE_FAILED;
+  } else {
+    *pFuncType = funcMgtBuiltins[*pFuncId].type;
   }
-  *pFuncType = funcMgtBuiltins[*pFuncId].type;
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (fmIsUserDefinedFunc(pFunc->funcId)) {
+    return getUdfResultType(pFunc);
+  }
+
   if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) {
     return TSDB_CODE_FAILED;
   }
@@ -77,7 +116,7 @@ int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
 }
 
 EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
-  if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) {
+  if (fmIsUserDefinedFunc(pFunc->funcId) || pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) {
     return FUNC_DATA_REQUIRED_DATA_LOAD;
   }
   if (NULL == funcMgtBuiltins[pFunc->funcId].dataRequiredFunc) {
@@ -87,7 +126,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
 }
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
-  if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
+  if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return TSDB_CODE_FAILED;
   }
   pFpSet->getEnv = funcMgtBuiltins[funcId].getEnvFunc;
@@ -98,7 +137,7 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
 }
 
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
-  if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
+  if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return TSDB_CODE_FAILED;
   }
   pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc;
@@ -142,6 +181,10 @@ bool fmIsMultiResFunc(int32_t funcId) {
   return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC);
 }
 
+bool fmIsUserDefinedFunc(int32_t funcId) {
+  return funcId > FUNC_UDF_ID_START_OFFSET_VAL;
+}
+
 void fmFuncMgtDestroy() {
   void* m = gFunMgtService.pFuncNameHashTable;
   if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) {

source/libs/function/src/tudf.c

@@ -200,10 +200,10 @@ int64_t gUdfTaskSeqNum = 0;
 
 enum {
   UDFC_STATE_INITAL = 0, // initial state
-  UDFC_STATE_STARTNG, // starting after startUdfService
+  UDFC_STATE_STARTNG, // starting after createUdfdProxy
   UDFC_STATE_READY, // started and begin to receive quests
   UDFC_STATE_RESTARTING, // udfd abnormal exit. cleaning up and restart.
-  UDFC_STATE_STOPPING, // stopping after stopUdfService
+  UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy
   UDFC_STATUS_FINAL, // stopped
 };
 int8_t gUdfcState = UDFC_STATE_INITAL;
@@ -929,7 +929,7 @@ void udfStopAsyncCb(uv_async_t *async) {
   }
 }
 
-int32_t startUdfd();
+int32_t udfcSpawnUdfd();
 
 void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) {
   //TODO: pipe close will be first received
@@ -944,12 +944,12 @@ void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) {
   if (gUdfcState == UDFC_STATE_READY) {
     gUdfcState = UDFC_STATE_RESTARTING;
     //TODO: asynchronous without blocking. how to do it
-    cleanUpUvTasks();
+    //cleanUpUvTasks();
-    startUdfd();
+    udfcSpawnUdfd();
   }
 }
 
-int32_t startUdfd() {
+int32_t udfcSpawnUdfd() {
   //TODO: path
   uv_process_options_t options = {0};
   static char path[256] = {0};
@@ -979,9 +979,6 @@ int32_t startUdfd() {
 void constructUdfService(void *argsThread) {
   uv_loop_init(&gUdfdLoop);
 
-  //TODO spawn error
-  startUdfd();
-
   uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb);
   uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb);
   uv_mutex_init(&gUdfTaskQueueMutex);
@@ -994,7 +991,7 @@ void constructUdfService(void *argsThread) {
 }
 
 
-int32_t startUdfService() {
+int32_t createUdfdProxy(int32_t dnodeId) {
   gUdfcState = UDFC_STATE_STARTNG;
   uv_barrier_init(&gUdfInitBarrier, 2);
   uv_thread_create(&gUdfLoopThread, constructUdfService, 0);
@@ -1002,12 +999,12 @@ int32_t startUdfService() {
   return 0;
 }
 
-int32_t stopUdfService() {
+int32_t destroyUdfdProxy(int32_t dnodeId) {
   gUdfcState = UDFC_STATE_STOPPING;
   uv_barrier_destroy(&gUdfInitBarrier);
-  if (gUdfcState == UDFC_STATE_STOPPING) {
+//  if (gUdfcState == UDFC_STATE_STOPPING) {
-    uv_process_kill(&gUdfdProcess, SIGINT);
+//    uv_process_kill(&gUdfdProcess, SIGINT);
-  }
+//  }
   uv_async_send(&gUdfLoopStopAsync);
   uv_thread_join(&gUdfLoopThread);
   uv_mutex_destroy(&gUdfTaskQueueMutex);

source/libs/function/src/udfd.c

@@ -12,10 +12,10 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
 #include "uv.h"
 #include "os.h"
-#include "tlog.h"
+#include "fnLog.h"
+#include "thash.h"
 
 #include "tudf.h"
 #include "tudfInt.h"
@@ -25,336 +25,377 @@
 #include "tmsg.h"
 #include "trpc.h"
 
-static uv_loop_t *loop;
+typedef struct SUdfdContext {
+  uv_loop_t *loop;
+  char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN];
+  void      *clientRpc;
+
+  uv_mutex_t udfsMutex;
+  SHashObj* udfsHash;
+
+  bool printVersion;
+} SUdfdContext;
+
+SUdfdContext global;
 
 typedef struct SUdfdUvConn {
-    uv_stream_t *client;
+  uv_stream_t *client;
-    char *inputBuf;
+  char        *inputBuf;
-    int32_t inputLen;
+  int32_t      inputLen;
-    int32_t inputCap;
+  int32_t      inputCap;
-    int32_t inputTotal;
+  int32_t      inputTotal;
 } SUdfdUvConn;
 
 typedef struct SUvUdfWork {
-    uv_stream_t *client;
+  uv_stream_t *client;
-    uv_buf_t input;
+  uv_buf_t     input;
-    uv_buf_t output;
+  uv_buf_t     output;
 } SUvUdfWork;
 
+typedef enum {
+  UDF_STATE_INIT = 0,
+  UDF_STATE_LOADING,
+  UDF_STATE_READY,
+  UDF_STATE_UNLOADING
+} EUdfState;
+
 typedef struct SUdf {
-    int32_t refCount;
+  int32_t refCount;
+  EUdfState state;
+  uv_mutex_t lock;
+  uv_cond_t condReady;
 
-    char name[16];
+  char   name[16];
-    int8_t type;
+  int8_t type;
+  char path[PATH_MAX];
 
-    uv_lib_t lib;
+  uv_lib_t              lib;
-    TUdfScalarProcFunc scalarProcFunc;
+  TUdfScalarProcFunc    scalarProcFunc;
-    TUdfFreeUdfColumnFunc freeUdfColumn;
+  TUdfFreeUdfColumnFunc freeUdfColumn;
 } SUdf;
 
-//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
+// TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
-//TODO: add private udf structure.
+// TODO: add private udf structure.
 typedef struct SUdfHandle {
-    SUdf *udf;
+  SUdf *udf;
 } SUdfHandle;
 
+int32_t udfdLoadUdf(char* udfName, SUdf* udf) {
+    strcpy(udf->name, udfName);
+
+    int   err = uv_dlopen(udf->path, &udf->lib);
+    if (err != 0) {
+      fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
+      // TODO set error
+    }
+    //TODO: find all the functions
+    char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
+    strcpy(normalFuncName, udfName);
+    uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc));
+    char  freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
+    char *freeSuffix = "_free";
+    strncpy(freeFuncName, normalFuncName, strlen(normalFuncName));
+    strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
+    uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
+    return 0;
+}
 
 void udfdProcessRequest(uv_work_t *req) {
-    SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data);
+  SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
-    SUdfRequest request = {0};
+  SUdfRequest request = {0};
-    decodeUdfRequest(uvUdf->input.base, &request);
+  decodeUdfRequest(uvUdf->input.base, &request);
 
-    switch (request.type) {
+  switch (request.type) {
-        case UDF_TASK_SETUP: {
+    case UDF_TASK_SETUP: {
-            debugPrint("%s", "process setup request");
+      //TODO: tracable id from client. connect, setup, call, teardown
-            SUdf *udf = taosMemoryMalloc(sizeof(SUdf));
+      fnInfo("%"PRId64" setup request. udf name: %s", request.seqNum, request.setup.udfName);
-            udf->refCount = 0;
+      SUdfSetupRequest *setup = &request.setup;
-            SUdfSetupRequest *setup = &request.setup;
-            strcpy(udf->name, setup->udfName);
-            //TODO: retrive udf info from mnode
-            char* path = "libudf1.so";
-            int err = uv_dlopen(path, &udf->lib);
-            if (err != 0) {
-                debugPrint("can not load library %s. error: %s", path, uv_strerror(err));
-                //TODO set error
-            }
 
-            char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
+      SUdf* udf = NULL;
-            strcpy(normalFuncName, setup->udfName);
+      uv_mutex_lock(&global.udfsMutex);
-	    //TODO error, multi-thread, same udf, lock it
+      SUdf** udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN);
-	    //TODO find all functions normal, init, destroy, normal, merge, finalize
+      if (*udfInHash) {
-            uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->scalarProcFunc));
+        ++(*udfInHash)->refCount;
-            char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
+        udf = *udfInHash;
-            char *freeSuffix = "_free";
+        uv_mutex_unlock(&global.udfsMutex);
-            strncpy(freeFuncName, normalFuncName, strlen(normalFuncName));
+      } else {
-            strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
+        SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf));
-            uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
+        udfNew->refCount = 1;
+        udfNew->state = UDF_STATE_INIT;
 
-            SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle));
+        uv_mutex_init(&udfNew->lock);
-            handle->udf = udf;
+        uv_cond_init(&udfNew->condReady);
-            udf->refCount++;
+        udf = udfNew;
-            //TODO: allocate private structure and call init function and set it to handle
+        taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew));
-            SUdfResponse rsp;
+        uv_mutex_unlock(&global.udfsMutex);
-            rsp.seqNum = request.seqNum;
+      }
-            rsp.type = request.type;
-            rsp.code = 0;
-            rsp.setupRsp.udfHandle = (int64_t) (handle);
-            int32_t len = encodeUdfResponse(NULL, &rsp);
-            rsp.msgLen = len;
-            void *bufBegin = taosMemoryMalloc(len);
-            void *buf = bufBegin;
-            encodeUdfResponse(&buf, &rsp);
 
-            uvUdf->output = uv_buf_init(bufBegin, len);
+      uv_mutex_lock(&udf->lock);
-
+      if (udf->state == UDF_STATE_INIT) {
-            taosMemoryFree(uvUdf->input.base);
+        udf->state = UDF_STATE_LOADING;
-            break;
+        udfdLoadUdf(setup->udfName, udf);
+        udf->state = UDF_STATE_READY;
+        uv_cond_broadcast(&udf->condReady);
+        uv_mutex_unlock(&udf->lock);
+      } else {
+        while (udf->state != UDF_STATE_READY) {
+          uv_cond_wait(&udf->condReady, &udf->lock);
         }
+        uv_mutex_unlock(&udf->lock);
+      }
+      SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle));
+      handle->udf = udf;
+      // TODO: allocate private structure and call init function and set it to handle
+      SUdfResponse rsp;
+      rsp.seqNum = request.seqNum;
+      rsp.type = request.type;
+      rsp.code = 0;
+      rsp.setupRsp.udfHandle = (int64_t)(handle);
+      int32_t len = encodeUdfResponse(NULL, &rsp);
+      rsp.msgLen = len;
+      void *bufBegin = taosMemoryMalloc(len);
+      void *buf = bufBegin;
+      encodeUdfResponse(&buf, &rsp);
 
-        case UDF_TASK_CALL: {
+      uvUdf->output = uv_buf_init(bufBegin, len);
-            debugPrint("%s", "process call request");
-            SUdfCallRequest *call = &request.call;
-            SUdfHandle *handle = (SUdfHandle *) (call->udfHandle);
-            SUdf *udf = handle->udf;
-
-            SUdfDataBlock input = {0};
-            convertDataBlockToUdfDataBlock(&call->block, &input);
-            SUdfColumn output = {0};
-	    //TODO: call different functions according to call type, for now just calar
-            if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
-              udf->scalarProcFunc(input, &output);
-            }
-
-            SUdfResponse  response = {0};
-            SUdfResponse *rsp = &response;
-            if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
-              rsp->seqNum = request.seqNum;
-              rsp->type = request.type;
-              rsp->code = 0;
-              SUdfCallResponse *subRsp = &rsp->callRsp;
-              subRsp->callType = call->callType;
-              convertUdfColumnToDataBlock(&output, &subRsp->resultData);
-            }
-
-            int32_t len = encodeUdfResponse(NULL, rsp);
-            rsp->msgLen = len;
-            void *bufBegin = taosMemoryMalloc(len);
-            void *buf = bufBegin;
-            encodeUdfResponse(&buf, rsp);
-            uvUdf->output = uv_buf_init(bufBegin, len);
-
-            //TODO: free
-            udf->freeUdfColumn(&output);
-
-            taosMemoryFree(uvUdf->input.base);
-            break;
-        }
-        case UDF_TASK_TEARDOWN: {
-            debugPrint("%s", "process teardown request");
-
-            SUdfTeardownRequest *teardown = &request.teardown;
-            SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle);
-            SUdf *udf = handle->udf;
-            udf->refCount--;
-            if (udf->refCount == 0) {
-                uv_dlclose(&udf->lib);
-                taosMemoryFree(udf);
-            }
-	    //TODO: call destroy and free udf private
-            taosMemoryFree(handle);
-
-            SUdfResponse response;
-            SUdfResponse *rsp = &response;
-            rsp->seqNum = request.seqNum;
-            rsp->type = request.type;
-            rsp->code = 0;
-            int32_t len = encodeUdfResponse(NULL, rsp);
-            rsp->msgLen = len;
-            void *bufBegin = taosMemoryMalloc(len);
-            void *buf = bufBegin;
-            encodeUdfResponse(&buf, rsp);
-            uvUdf->output = uv_buf_init(bufBegin, len);
-
-            taosMemoryFree(uvUdf->input.base);
-            break;
-        }
-        default: {
-            break;
-        }
 
+      taosMemoryFree(uvUdf->input.base);
+      break;
     }
 
+    case UDF_TASK_CALL: {
+      SUdfCallRequest *call = &request.call;
+      fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle);
+      SUdfHandle      *handle = (SUdfHandle *)(call->udfHandle);
+      SUdf            *udf = handle->udf;
+
+      SUdfDataBlock input = {0};
+      convertDataBlockToUdfDataBlock(&call->block, &input);
+      SUdfColumn output = {0};
+      // TODO: call different functions according to call type, for now just calar
+      if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
+        udf->scalarProcFunc(input, &output);
+      }
+
+      SUdfResponse  response = {0};
+      SUdfResponse *rsp = &response;
+      if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
+        rsp->seqNum = request.seqNum;
+        rsp->type = request.type;
+        rsp->code = 0;
+        SUdfCallResponse *subRsp = &rsp->callRsp;
+        subRsp->callType = call->callType;
+        convertUdfColumnToDataBlock(&output, &subRsp->resultData);
+      }
+
+      int32_t len = encodeUdfResponse(NULL, rsp);
+      rsp->msgLen = len;
+      void *bufBegin = taosMemoryMalloc(len);
+      void *buf = bufBegin;
+      encodeUdfResponse(&buf, rsp);
+      uvUdf->output = uv_buf_init(bufBegin, len);
+
+      // TODO: free udf column
+      udf->freeUdfColumn(&output);
+
+      taosMemoryFree(uvUdf->input.base);
+      break;
+    }
+    case UDF_TASK_TEARDOWN: {
+      SUdfTeardownRequest *teardown = &request.teardown;
+      fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle)
+      SUdfHandle          *handle = (SUdfHandle *)(teardown->udfHandle);
+      SUdf                *udf = handle->udf;
+      bool unloadUdf = false;
+      uv_mutex_lock(&global.udfsMutex);
+      udf->refCount--;
+      if (udf->refCount == 0) {
+        unloadUdf = true;
+        taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN);
+      }
+      uv_mutex_unlock(&global.udfsMutex);
+      if (unloadUdf) {
+        uv_cond_destroy(&udf->condReady);
+        uv_mutex_destroy(&udf->lock);
+        uv_dlclose(&udf->lib);
+        taosMemoryFree(udf);
+      }
+      // TODO: call destroy and free udf private
+      taosMemoryFree(handle);
+
+      SUdfResponse  response;
+      SUdfResponse *rsp = &response;
+      rsp->seqNum = request.seqNum;
+      rsp->type = request.type;
+      rsp->code = 0;
+      int32_t len = encodeUdfResponse(NULL, rsp);
+      rsp->msgLen = len;
+      void *bufBegin = taosMemoryMalloc(len);
+      void *buf = bufBegin;
+      encodeUdfResponse(&buf, rsp);
+      uvUdf->output = uv_buf_init(bufBegin, len);
+
+      taosMemoryFree(uvUdf->input.base);
+      break;
+    }
+    default: {
+      break;
+    }
+  }
 }
 
 void udfdOnWrite(uv_write_t *req, int status) {
-    debugPrint("%s", "server after writing to pipe");
+  SUvUdfWork *work = (SUvUdfWork *)req->data;
-    if (status < 0) {
+  if (status < 0) {
-        debugPrint("Write error %s", uv_err_name(status));
+    //TODO:log error and process it.
-    }
+  }
-    SUvUdfWork *work = (SUvUdfWork *) req->data;
+  fnDebug("send response. length:%zu, status: %s", work->output.len, uv_err_name(status));
-    debugPrint("\tlength: %zu", work->output.len);
+  taosMemoryFree(work->output.base);
-    taosMemoryFree(work->output.base);
+  taosMemoryFree(work);
-    taosMemoryFree(work);
+  taosMemoryFree(req);
-    taosMemoryFree(req);
 }
 
-
 void udfdSendResponse(uv_work_t *work, int status) {
-    debugPrint("%s", "send response");
+  SUvUdfWork *udfWork = (SUvUdfWork *)(work->data);
-    SUvUdfWork *udfWork = (SUvUdfWork *) (work->data);
 
-    uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t));
+  uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t));
-    write_req->data = udfWork;
+  write_req->data = udfWork;
-    uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite);
+  uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite);
 
-    taosMemoryFree(work);
+  taosMemoryFree(work);
 }
 
 void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
-    debugPrint("%s", "server allocate buffer for read");
+  SUdfdUvConn *ctx = handle->data;
-    SUdfdUvConn *ctx = handle->data;
+  int32_t      msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
-    int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
+  if (ctx->inputCap == 0) {
-    if (ctx->inputCap == 0) {
+    ctx->inputBuf = taosMemoryMalloc(msgHeadSize);
-        ctx->inputBuf = taosMemoryMalloc(msgHeadSize);
+    if (ctx->inputBuf) {
-        if (ctx->inputBuf) {
+      ctx->inputLen = 0;
-            ctx->inputLen = 0;
+      ctx->inputCap = msgHeadSize;
-            ctx->inputCap = msgHeadSize;
+      ctx->inputTotal = -1;
-            ctx->inputTotal = -1;
 
-            buf->base = ctx->inputBuf;
+      buf->base = ctx->inputBuf;
-            buf->len = ctx->inputCap;
+      buf->len = ctx->inputCap;
-        } else {
-            //TODO: log error
-            buf->base = NULL;
-            buf->len = 0;
-        }
     } else {
-        ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap;
+      // TODO: log error
-        void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap);
+      buf->base = NULL;
-        if (inputBuf) {
+      buf->len = 0;
-            ctx->inputBuf = inputBuf;
-            buf->base = ctx->inputBuf + ctx->inputLen;
-            buf->len = ctx->inputCap - ctx->inputLen;
-        } else {
-            //TODO: log error
-            buf->base = NULL;
-            buf->len = 0;
-        }
     }
-    debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal);
+  } else {
-
+    ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap;
+    void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap);
+    if (inputBuf) {
+      ctx->inputBuf = inputBuf;
+      buf->base = ctx->inputBuf + ctx->inputLen;
+      buf->len = ctx->inputCap - ctx->inputLen;
+    } else {
+      // TODO: log error
+      buf->base = NULL;
+      buf->len = 0;
+    }
+  }
+  fnDebug("allocate buf. input buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal);
 }
 
 bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) {
-    if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) {
+  if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) {
-        pipe->inputTotal = *(int32_t *) (pipe->inputBuf);
+    pipe->inputTotal = *(int32_t *)(pipe->inputBuf);
-    }
+  }
-    if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) {
+  if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) {
-        return true;
+    fnDebug("receive request complete. length %d", pipe->inputLen);
-    }
+    return true;
-    return false;
+  }
+  return false;
 }
 
 void udfdHandleRequest(SUdfdUvConn *conn) {
-    uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t));
+  uv_work_t  *work = taosMemoryMalloc(sizeof(uv_work_t));
-    SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork));
+  SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork));
-    udfWork->client = conn->client;
+  udfWork->client = conn->client;
-    udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen);
+  udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen);
-    conn->inputBuf = NULL;
+  conn->inputBuf = NULL;
-    conn->inputLen = 0;
+  conn->inputLen = 0;
-    conn->inputCap = 0;
+  conn->inputCap = 0;
-    conn->inputTotal = -1;
+  conn->inputTotal = -1;
-    work->data = udfWork;
+  work->data = udfWork;
-    uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse);
+  uv_queue_work(global.loop, work, udfdProcessRequest, udfdSendResponse);
 }
 
 void udfdPipeCloseCb(uv_handle_t *pipe) {
-    SUdfdUvConn *conn = pipe->data;
+  SUdfdUvConn *conn = pipe->data;
-    taosMemoryFree(conn->client);
+  taosMemoryFree(conn->client);
-    taosMemoryFree(conn->inputBuf);
+  taosMemoryFree(conn->inputBuf);
-    taosMemoryFree(conn);
+  taosMemoryFree(conn);
 }
 
-void udfdUvHandleError(SUdfdUvConn *conn) {
+void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); }
-    uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb);
-}
 
 void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
-    debugPrint("%s, nread: %zd", "read from pipe", nread);
+  fnDebug("udf read %zu bytes from client", nread);
+  if (nread == 0) return;
 
-    if (nread == 0) return;
+  SUdfdUvConn *conn = client->data;
 
-    SUdfdUvConn *conn = client->data;
+  if (nread > 0) {
-
+    conn->inputLen += nread;
-    if (nread > 0) {
+    if (isUdfdUvMsgComplete(conn)) {
-        conn->inputLen += nread;
+      udfdHandleRequest(conn);
-        if (isUdfdUvMsgComplete(conn)) {
+    } else {
-            udfdHandleRequest(conn);
+      // log error or continue;
-        } else {
-            //log error or continue;
-        }
-        return;
     }
+    return;
+  }
 
-    if (nread < 0) {
+  if (nread < 0) {
-        debugPrint("Read error %s", uv_err_name(nread));
+    fnDebug("Receive error %s", uv_err_name(nread));
-        if (nread == UV_EOF) {
+    if (nread == UV_EOF) {
-            //TODO check more when close
+      // TODO check more when close
-        } else {
+    } else {
-        }
-        udfdUvHandleError(conn);
     }
+    udfdUvHandleError(conn);
+  }
 }
 
 void udfdOnNewConnection(uv_stream_t *server, int status) {
-    debugPrint("%s", "on new connection");
+  fnDebug("new connection");
-    if (status < 0) {
+  if (status < 0) {
-        // TODO
+    // TODO
-        return;
+    return;
-    }
+  }
 
-    uv_pipe_t *client = (uv_pipe_t *) taosMemoryMalloc(sizeof(uv_pipe_t));
+  uv_pipe_t *client = (uv_pipe_t *)taosMemoryMalloc(sizeof(uv_pipe_t));
-    uv_pipe_init(loop, client, 0);
+  uv_pipe_init(global.loop, client, 0);
-    if (uv_accept(server, (uv_stream_t *) client) == 0) {
+  if (uv_accept(server, (uv_stream_t *)client) == 0) {
-        SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn));
+    SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn));
-        ctx->client = (uv_stream_t *) client;
+    ctx->client = (uv_stream_t *)client;
-        ctx->inputBuf = 0;
+    ctx->inputBuf = 0;
-        ctx->inputLen = 0;
+    ctx->inputLen = 0;
-        ctx->inputCap = 0;
+    ctx->inputCap = 0;
-        client->data = ctx;
+    client->data = ctx;
-        ctx->client = (uv_stream_t *) client;
+    ctx->client = (uv_stream_t *)client;
-        uv_read_start((uv_stream_t *) client, udfdAllocBuffer, udfdPipeRead);
+    uv_read_start((uv_stream_t *)client, udfdAllocBuffer, udfdPipeRead);
-    } else {
+  } else {
-        uv_close((uv_handle_t *) client, NULL);
+    uv_close((uv_handle_t *)client, NULL);
-    }
+  }
 }
 
 void removeListeningPipe(int sig) {
-    uv_fs_t req;
+  uv_fs_t req;
-    uv_fs_unlink(loop, &req, "udf.sock", NULL);
+  uv_fs_unlink(global.loop, &req, "udf.sock", NULL);
-    exit(0);
+  exit(0);
 }
 
-typedef struct SServerContext {
+void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
-  void *clientRpc;
-} SUdfdContext;
 
-
+int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char* udfName, SUdf* udf) {
-void udfdProcessRpcRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)  {
-
-  return;
-}
-
-int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int32_t numOfUdfs) {
   SRetrieveFuncReq retrieveReq = {0};
   retrieveReq.numOfFuncs = 1;
   retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
-  for (int32_t i = 0; i < numOfUdfs; ++i) {
+  taosArrayPush(retrieveReq.pFuncNames, udfName);
-    taosArrayPush(retrieveReq.pFuncNames, udfNames[i]);
-  }
 
   int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
-  void*   pReq = rpcMallocCont(contLen);
+  void   *pReq = rpcMallocCont(contLen);
   tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
   taosArrayDestroy(retrieveReq.pFuncNames);
 
@@ -368,66 +409,176 @@ int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int3
   SRetrieveFuncRsp retrieveRsp = {0};
   tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
 
-  SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0);
+  SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
 
+  char path[PATH_MAX] = {0};
+  taosGetTmpfilePath("/tmp", "libudf", path);
+  TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  // TODO check for failure of flush to disk
+  taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
+  taosCloseFile(&file);
+  strncpy(udf->path, path, strlen(path));
   taosArrayDestroy(retrieveRsp.pFuncInfos);
 
   rpcFreeCont(rpcRsp.pCont);
   return 0;
 }
 
-int32_t openUdfdClientRpc(SUdfdContext *ctx) {
+int32_t udfdOpenClientRpc() {
   char *pass = "taosdata";
   char *user = "root";
-  char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
+  char  secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
-  taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt);
+  taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt);
   SRpcInit rpcInit = {0};
-  rpcInit.label = (char*)"UDFD";
+  rpcInit.label = (char *)"UDFD";
   rpcInit.numOfThreads = 1;
   rpcInit.cfp = udfdProcessRpcRsp;
   rpcInit.sessions = 1024;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.idleTime = 30 * 1000;
-  rpcInit.parent = ctx;
+  rpcInit.parent = &global;
 
-  rpcInit.user = (char*)user;
+  rpcInit.user = (char *)user;
-  rpcInit.ckey = (char*)"key";
+  rpcInit.ckey = (char *)"key";
-  rpcInit.secret = (char*)secretEncrypt;
+  rpcInit.secret = (char *)secretEncrypt;
   rpcInit.spi = 1;
 
-  ctx->clientRpc = rpcOpen(&rpcInit);
+  global.clientRpc = rpcOpen(&rpcInit);
 
   return 0;
 }
 
-int32_t closeUdfdClientRpc(SUdfdContext *ctx) {
+int32_t udfdCloseClientRpc() {
-  rpcClose(ctx->clientRpc);
+  rpcClose(global.clientRpc);
+  return 0;
+}
+
+static void udfdPrintVersion() {
+#ifdef TD_ENTERPRISE
+  char *releaseName = "enterprise";
+#else
+  char *releaseName = "community";
+#endif
+  printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
+  printf("gitinfo: %s\n", gitinfo);
+  printf("buildInfo: %s\n", buildinfo);
+}
+
+static int32_t udfdParseArgs(int32_t argc, char *argv[]) {
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("config file path overflow");
+          return -1;
+        }
+        tstrncpy(configDir, argv[i], PATH_MAX);
+      } else {
+        printf("'-c' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-V") == 0) {
+      global.printVersion = true;
+    } else {
+    }
+  }
 
   return 0;
 }
 
-int main() {
+static int32_t udfdInitLog() {
-    debugPrint("libuv version: %x", UV_VERSION_HEX);
+  char logName[12] = {0};
-
+  snprintf(logName, sizeof(logName), "%slog", "udfd");
-    loop = uv_default_loop();
+  return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
-    uv_fs_t req;
+}
-    uv_fs_unlink(loop, &req, "udf.sock", NULL);
+
-
+static int32_t udfdUvInit() {
-    uv_pipe_t server;
+  uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t));
-    uv_pipe_init(loop, &server, 0);
+  if (loop) {
-
+    uv_loop_init(loop);
-    signal(SIGINT, removeListeningPipe);
+  }
-
+  global.loop = loop;
-    int r;
+  char dnodeId[8] = {0};
-    if ((r = uv_pipe_bind(&server, "udf.sock"))) {
+  size_t dnodeIdSize;
-        debugPrint("Bind error %s\n", uv_err_name(r));
+  uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
-        removeListeningPipe(0);
+  char listenPipeName[32] = {0};
-        return 1;
+  snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
-    }
+  strcpy(global.listenPipeName, listenPipeName);
-    if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) {
+
-        debugPrint("Listen error %s", uv_err_name(r));
+  uv_fs_t req;
-        return 2;
+  uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
-    }
+
-    uv_run(loop, UV_RUN_DEFAULT);
+  uv_pipe_t server;
-    uv_loop_close(loop);
+  uv_pipe_init(global.loop, &server, 0);
+
+  signal(SIGINT, removeListeningPipe);
+
+  int r;
+  fnInfo("bind to pipe %s", global.listenPipeName);
+  if ((r = uv_pipe_bind(&server, listenPipeName))) {
+    fnError("Bind error %s", uv_err_name(r));
+    removeListeningPipe(0);
+    return -1;
+  }
+  if ((r = uv_listen((uv_stream_t *)&server, 128, udfdOnNewConnection))) {
+    fnError("Listen error %s", uv_err_name(r));
+    removeListeningPipe(0);
+    return -2;
+  }
+  return 0;
+}
+
+static int32_t udfdRun() {
+  global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  uv_mutex_init(&global.udfsMutex);
+
+  //TOOD: client rpc to fetch udf function info from mnode
+  if (udfdOpenClientRpc() != 0) {
+    fnError("open rpc connection to mnode failure");
+    return -1;
+  }
+
+  if (udfdUvInit() != 0) {
+    fnError("uv init failure");
+    return -2;
+  }
+
+  fnInfo("start the udfd");
+  int code = uv_run(global.loop, UV_RUN_DEFAULT);
+  fnInfo("udfd stopped. result: %s", uv_err_name(code));
+  int codeClose = uv_loop_close(global.loop);
+  fnDebug("uv loop close. result: %s", uv_err_name(codeClose));
+  udfdCloseClientRpc();
+  uv_mutex_destroy(&global.udfsMutex);
+  taosHashCleanup(global.udfsHash);
+  return code;
+}
+
+int main(int argc, char* argv[]) {
+  if (!taosCheckSystemIsSmallEnd()) {
+    printf("failed to start since on non-small-end machines\n");
+    return -1;
+  }
+
+  if (udfdParseArgs(argc, argv) != 0) {
+    printf("failed to start since parse args error\n");
+    return -1;
+  }
+
+  if (global.printVersion) {
+    udfdPrintVersion();
+    return 0;
+  }
+
+  if (udfdInitLog() != 0) {
+    printf("failed to start since init log error\n");
+    return -1;
+  }
+
+  if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) {
+    fnError("failed to start since read config error");
+    return -1;
+  }
+
+  return udfdRun();
 }

source/libs/function/test/runUdf.c

@@ -8,7 +8,7 @@
 #include "tdatablock.h"
 
 int main(int argc, char *argv[]) {
-    startUdfService();
+  createUdfdProxy(1);
     uv_sleep(1000);
     char path[256] = {0};
     size_t cwdSize = 256;
@@ -53,5 +53,5 @@ int main(int argc, char *argv[]) {
     }
     teardownUdf(handle);
 
-    stopUdfService();
+    destroyUdfdProxy(1);
 }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -149,7 +149,9 @@ SNodeptr nodesMakeNode(ENodeType type) {
     case QUERY_NODE_RESET_QUERY_CACHE_STMT:
       return makeNode(type, sizeof(SNode));
     case QUERY_NODE_COMPACT_STMT:
+      break;
     case QUERY_NODE_CREATE_FUNCTION_STMT:
+      return makeNode(type, sizeof(SCreateFunctionStmt));
     case QUERY_NODE_DROP_FUNCTION_STMT:
       break;
     case QUERY_NODE_CREATE_STREAM_STMT:
@@ -167,6 +169,7 @@ SNodeptr nodesMakeNode(ENodeType type) {
     case QUERY_NODE_SHOW_QNODES_STMT:
     case QUERY_NODE_SHOW_SNODES_STMT:
     case QUERY_NODE_SHOW_BNODES_STMT:
+    case QUERY_NODE_SHOW_CLUSTER_STMT:
     case QUERY_NODE_SHOW_DATABASES_STMT:
     case QUERY_NODE_SHOW_FUNCTIONS_STMT:
     case QUERY_NODE_SHOW_INDEXES_STMT:
@@ -1006,6 +1009,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
     case OP_TYPE_NOT_LIKE:
     case OP_TYPE_MATCH:
     case OP_TYPE_NMATCH:
+    case OP_TYPE_JSON_CONTAINS:
     case OP_TYPE_IS_NULL:
     case OP_TYPE_IS_NOT_NULL:
     case OP_TYPE_IS_TRUE:
@@ -1024,7 +1028,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
 bool nodesIsJsonOp(const SOperatorNode* pOp) {
   switch (pOp->opType) {
     case OP_TYPE_JSON_GET_VALUE:
-    case OP_TYPE_JSON_CONTAINS:
       return true;
     default:
       break;

source/libs/parser/inc/parAst.h

@@ -167,7 +167,7 @@ SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions,
 SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable);
 SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt);
 SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups);
-SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize);
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize);
 SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName);
 SNode* createStreamOptions(SAstCreateContext* pCxt);
 SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, SNode* pOptions, SNode* pQuery);

source/libs/parser/inc/parInsertData.h

@@ -89,7 +89,7 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
          (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1);
 }
 
-static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx,
+static FORCE_INLINE void getSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx,
                                              int32_t *toffset, col_id_t *colIdx) {
   col_id_t schemaIdx = 0;
   if (IS_DATA_COL_ORDERED(spd)) {
@@ -131,7 +131,6 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks*
 int32_t schemaIdxCompar(const void *lhs, const void *rhs);
 int32_t boundIdxCompar(const void *lhs, const void *rhs);
 void    setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols);
-void destroyBoundColumnInfo(SParsedDataColInfo* pColList);
 void destroyBlockArrayList(SArray* pDataBlockList);
 void destroyBlockHashmap(SHashObj* pDataBlockHash);
 int  initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo);
@@ -139,5 +138,7 @@ int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t
 int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
     const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq);
 int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks);
+int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq);
+int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize);
 
 #endif  // TDENGINE_DATABLOCKMGT_H

source/libs/parser/inc/parInt.h

@@ -21,6 +21,8 @@ extern "C" {
 #endif
 
 #include "parser.h"
+#include "parToken.h"
+#include "parUtil.h"
 
 int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery);
 int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);

source/libs/parser/inc/parUtil.h

@@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta);
 int32_t  getNumOfColumns(const STableMeta* pTableMeta);
 int32_t  getNumOfTags(const STableMeta* pTableMeta);
 STableComInfo getTableInfo(const STableMeta* pTableMeta);
+int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId);
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
 

source/libs/parser/inc/sql.y

@@ -346,6 +346,7 @@ cmd ::= SHOW TOPICS.
 cmd ::= SHOW VARIABLES.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT, NULL, NULL); }
 cmd ::= SHOW BNODES.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT, NULL, NULL); }
 cmd ::= SHOW SNODES.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT, NULL, NULL); }
+cmd ::= SHOW CLUSTER.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT, NULL, NULL); }
 
 db_name_cond_opt(A) ::= .                                                         { A = createDefaultDatabaseCondValue(pCxt); }
 db_name_cond_opt(A) ::= db_name(B) NK_DOT.                                        { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); }
@@ -413,8 +414,8 @@ explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C).
 cmd ::= COMPACT VNODES IN NK_LP integer_list(A) NK_RP.                            { pCxt->pRootNode = createCompactStmt(pCxt, A); }
 
 /************************************************ create/drop function ************************************************/
-cmd ::= CREATE agg_func_opt(A) FUNCTION function_name(B) 
+cmd ::= CREATE agg_func_opt(A) FUNCTION not_exists_opt(F) function_name(B) 
-  AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E).                         { pCxt->pRootNode = createCreateFunctionStmt(pCxt, A, &B, &C, D, E); }
+  AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E).                         { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E); }
 cmd ::= DROP FUNCTION function_name(A).                                           { pCxt->pRootNode = createDropFunctionStmt(pCxt, &A); }
 
 %type agg_func_opt                                                                { bool }
@@ -836,6 +837,8 @@ query_expression(A) ::=
 query_expression_body(A) ::= query_primary(B).                                    { A = B; }
 query_expression_body(A) ::=
   query_expression_body(B) UNION ALL query_expression_body(D).                    { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); }
+query_expression_body(A) ::=
+  query_expression_body(B) UNION query_expression_body(D).                        { A = createSetOperator(pCxt, SET_OP_TYPE_UNION, B, D); }
 
 query_primary(A) ::= query_specification(B).                                      { A = B; }
 //query_primary(A) ::=

source/libs/parser/src/parAstCreater.c

@@ -48,6 +48,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
 }
 
 static void trimEscape(SToken* pName) {
+  // todo need to deal with `ioo``ii` -> ioo`ii
   if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) {
     pName->z += 1;
     pName->n -= 2;
@@ -1181,10 +1182,21 @@ SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) {
   return pStmt;
 }
 
-SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) {
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt,
-  SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT);
+    bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) {
+  if (pLibPath->n <= 2) {
+    pCxt->valid = false;
+    return NULL;
+  }
+  SCreateFunctionStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT);
   CHECK_OUT_OF_MEM(pStmt);
-  return pStmt;
+  pStmt->ignoreExists = ignoreExists;
+  strncpy(pStmt->funcName, pFuncName->z, pFuncName->n);
+  pStmt->isAgg = aggFunc;
+  strncpy(pStmt->libraryPath, pLibPath->z + 1, pLibPath->n - 2);
+  pStmt->outputDt = dataType;
+  pStmt->bufSize = bufSize;
+  return (SNode*)pStmt;
 }
 
 SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName) {

source/libs/parser/src/parInsert.c

@@ -40,29 +40,21 @@
     sToken = tStrGetToken(pSql, &index, false);  \
   } while (0)
 
-#define CHECK_CODE(expr)             \
-  do {                               \
-    int32_t code = expr;             \
-    if (TSDB_CODE_SUCCESS != code) { \
-      return code;                   \
-    }                                \
-  } while (0)
-
 typedef struct SInsertParseContext {
-  SParseContext*     pComCxt;             // input
+  SParseContext* pComCxt;       // input
-  char*              pSql;                // input
+  char          *pSql;          // input
-  SMsgBuf            msg;                 // input
+  SMsgBuf        msg;           // input
-  STableMeta*        pTableMeta;          // each table
+  STableMeta* pTableMeta;       // each table
-  SParsedDataColInfo tags;                // each table
+  SParsedDataColInfo tags;      // each table
-  SKVRowBuilder      tagsBuilder;         // each table
+  SKVRowBuilder tagsBuilder;    // each table
-  SVCreateTbReq      createTblReq;        // each table
+  SVCreateTbReq createTblReq;   // each table
-  SHashObj*          pVgroupsHashObj;     // global
+  SHashObj* pVgroupsHashObj;    // global
-  SHashObj*          pTableBlockHashObj;  // global
+  SHashObj* pTableBlockHashObj; // global
-  SHashObj*          pSubTableHashObj;    // global
+  SHashObj* pSubTableHashObj;   // global
-  SArray*            pTableDataBlocks;    // global
+  SArray* pVgDataBlocks;        // global
-  SArray*            pVgDataBlocks;       // global
+  int32_t totalNum;
-  int32_t            totalNum;
   SVnodeModifOpStmt* pOutput;
+  SStmtCallback* pStmtCb;
 } SInsertParseContext;
 
 typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
@@ -70,6 +62,29 @@ typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t
 static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
 static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
 
+typedef struct SKvParam {
+  SKVRowBuilder *builder;
+  SSchema       *schema;
+  char           buf[TSDB_MAX_TAGS_LEN];
+} SKvParam;
+
+typedef struct SMemParam {
+  SRowBuilder* rb;
+  SSchema*     schema;
+  int32_t      toffset;
+  col_id_t     colIdx;
+} SMemParam;
+
+
+#define CHECK_CODE(expr) \
+  do { \
+    int32_t code = expr; \
+    if (TSDB_CODE_SUCCESS != code) { \
+      return code; \
+    } \
+  } while (0)
+
+
 static int32_t skipInsertInto(SInsertParseContext* pCxt) {
   SToken sToken;
   NEXT_TOKEN(pCxt->pSql, sToken);
@@ -163,7 +178,8 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) {
+
+static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) {
   const char* msg1 = "name too long";
   const char* msg2 = "invalid database name";
   const char* msg3 = "db is not specified";
@@ -179,7 +195,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar
     strncpy(name, pTableName->z, dbLen);
     dbLen = strdequote(name);
 
-    code = tNameSetDbName(pName, pParseCtx->acctId, name, dbLen);
+    code = tNameSetDbName(pName, acctId, name, dbLen);
     if (code != TSDB_CODE_SUCCESS) {
       return buildInvalidOperationMsg(pMsgBuf, msg1);
     }
@@ -204,11 +220,11 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar
     strncpy(name, pTableName->z, pTableName->n);
     strdequote(name);
 
-    if (pParseCtx->db == NULL) {
+    if (dbName == NULL) {
       return buildInvalidOperationMsg(pMsgBuf, msg3);
     }
 
-    code = tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db));
+    code = tNameSetDbName(pName, acctId, dbName, strlen(dbName));
     if (code != TSDB_CODE_SUCCESS) {
       code = buildInvalidOperationMsg(pMsgBuf, msg2);
       return code;
@@ -225,8 +241,8 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar
 
 static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool isStb) {
   SParseContext* pBasicCtx = pCxt->pComCxt;
-  SName          name = {0};
+  SName name = {0};
-  createSName(&name, pTname, pBasicCtx, &pCxt->msg);
+  createSName(&name, pTname, pBasicCtx->acctId, pBasicCtx->db, &pCxt->msg);  
   if (isStb) {
     CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name,
                                     &pCxt->pTableMeta));
@@ -299,7 +315,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t checkTimestamp(STableDataBlocks* pDataBlocks, const char* start) {
+int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) {
   // once the data block is disordered, we do NOT keep previous timestamp any more
   if (!pDataBlocks->ordered) {
     return TSDB_CODE_SUCCESS;
@@ -397,35 +413,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
     return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
   }
 
-  if (IS_NUMERIC_TYPE(type) && pToken->n == 0) {
-    return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
-  }
-
   // Remove quotation marks
   if (TK_NK_STRING == pToken->type) {
     if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
       return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
     }
 
-    // delete escape character: \\, \', \"
+    int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
-    char    delim = pToken->z[0];
-    int32_t cnt = 0;
-    int32_t j = 0;
-    for (uint32_t k = 1; k < pToken->n - 1; ++k) {
-      if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
-        tmpTokenBuf[j] = pToken->z[k + 1];
-        cnt++;
-        j++;
-        k++;
-        continue;
-      }
-      tmpTokenBuf[j] = pToken->z[k];
-      j++;
-    }
-
-    tmpTokenBuf[j] = 0;
     pToken->z = tmpTokenBuf;
-    pToken->n -= 2 + cnt;
+    pToken->n = len;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -461,7 +457,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
 
   if (isNullStr(pToken)) {
     if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
-      return buildSyntaxErrMsg(pMsgBuf, "primary timestamp can not be null", pToken->z);
+      return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z);
     }
 
     return func(pMsgBuf, NULL, 0, param);
@@ -603,6 +599,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
       return func(pMsgBuf, pToken->z, pToken->n, param);
     }
 
+    case TSDB_DATA_TYPE_JSON: {
+      if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+        return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z);
+      }
+      return func(pMsgBuf, pToken->z, pToken->n, param);
+    }
+
     case TSDB_DATA_TYPE_TIMESTAMP: {
       int64_t tmpVal;
       if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
@@ -616,13 +619,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
   return TSDB_CODE_FAILED;
 }
 
-typedef struct SMemParam {
-  SRowBuilder* rb;
-  SSchema*     schema;
-  int32_t      toffset;
-  col_id_t     colIdx;
-} SMemParam;
-
 static FORCE_INLINE int32_t MemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) {
   SMemParam*   pa = (SMemParam*)param;
   SRowBuilder* rb = pa->rb;
@@ -723,27 +719,32 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
     qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
   }
 
-  memset(&pColList->boundColumns[pColList->numOfBound], 0,
+  if(pColList->numOfCols > pColList->numOfBound){
-         sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
+    memset(&pColList->boundColumns[pColList->numOfBound], 0,
+           sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
+  }
 
   return TSDB_CODE_SUCCESS;
 }
 
-typedef struct SKvParam {
+static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) {
-  SKVRowBuilder* builder;
+  SKvParam* pa = (SKvParam*) param;
-  SSchema*       schema;
-  char           buf[TSDB_MAX_TAGS_LEN];
-} SKvParam;
-
-static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) {
-  SKvParam* pa = (SKvParam*)param;
 
   int8_t  type = pa->schema->type;
   int16_t colId = pa->schema->colId;
 
+  if(TSDB_DATA_TYPE_JSON == type){
+    return parseJsontoTagData(value, pa->builder, pMsgBuf, colId);
+  }
+
+  if (value == NULL) {  // it is a null data
+    // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx);
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (TSDB_DATA_TYPE_BINARY == type) {
     STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
-    tdAddColToKVRow(pa->builder, colId, type, pa->buf);
+    tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
   } else if (TSDB_DATA_TYPE_NCHAR == type) {
     // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
     int32_t output = 0;
@@ -751,25 +752,24 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi
       char buf[512] = {0};
       snprintf(buf, tListLen(buf), "%s", strerror(errno));
       return buildSyntaxErrMsg(pMsgBuf, buf, value);
-      ;
     }
 
     varDataSetLen(pa->buf, output);
-    tdAddColToKVRow(pa->builder, colId, type, pa->buf);
+    tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
   } else {
-    tdAddColToKVRow(pa->builder, colId, type, value);
+    tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]);
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, SKVRow row) {
+static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const SName* pName, SKVRow row, int64_t suid) {
   char dbFName[TSDB_DB_FNAME_LEN] = {0};
   tNameGetFullDbName(pName, dbFName);
-  pCxt->createTblReq.type = TD_CHILD_TABLE;
+  pTbReq->type = TD_CHILD_TABLE;
-  pCxt->createTblReq.name = strdup(pName->tname);
+  pTbReq->name = strdup(pName->tname);
-  pCxt->createTblReq.ctbCfg.suid = pCxt->pTableMeta->suid;
+  pTbReq->ctbCfg.suid = suid;
-  pCxt->createTblReq.ctbCfg.pTag = row;
+  pTbReq->ctbCfg.pTag = row;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -781,23 +781,42 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
   }
 
   SKvParam param = {.builder = &pCxt->tagsBuilder};
-  SToken   sToken;
+  SToken sToken;
-  char     tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0};  // used for deleting Escape character: \\, \', \"
+  bool isParseBindParam = false;  
+  char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0};  // used for deleting Escape character: \\, \', \"
   for (int i = 0; i < pCxt->tags.numOfBound; ++i) {
     NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
-    SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1];  // colId starts with 1
+
+    if (sToken.type == TK_NK_QUESTION) {
+      isParseBindParam = true;
+      if (NULL == pCxt->pStmtCb) {
+        return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
+      }
+
+      continue;
+    }
+
+    if (isParseBindParam) {
+      return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values");
+    }
+    
+    SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1
     param.schema = pTagSchema;
     CHECK_CODE(
         parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, &param, &pCxt->msg));
   }
 
+  if (isParseBindParam) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder);
   if (NULL == row) {
     return buildInvalidOperationMsg(&pCxt->msg, "tag value expected");
   }
   tdSortKVRowByColIdx(row);
 
-  return buildCreateTbReq(pCxt, pName, row);
+  return buildCreateTbReq(&pCxt->createTblReq, pName, row, pCxt->pTableMeta->suid);
 }
 
 static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) {
@@ -821,7 +840,7 @@ static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, S
 // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
 static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) {
   SName name;
-  createSName(&name, pTbnameToken, pCxt->pComCxt, &pCxt->msg);
+  createSName(&name, pTbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
   char tbFName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(&name, tbFName);
   int32_t      len = strlen(tbFName);
@@ -866,8 +885,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken)
   return TSDB_CODE_SUCCESS;
 }
 
-static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len,
+static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow, char* tmpTokenBuf) {
-                       char* tmpTokenBuf) {
   SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo;
   SRowBuilder*        pBuilder = &pDataBlocks->rowBuilder;
   STSRow*             row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size);  // skip the SSubmitBlk header
@@ -882,8 +900,22 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
   for (int i = 0; i < spd->numOfBound; ++i) {
     NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
     SSchema* pSchema = &schema[spd->boundColumns[i] - 1];
+
+    if (sToken.type == TK_NK_QUESTION) {
+      isParseBindParam = true;
+      if (NULL == pCxt->pStmtCb) {
+        return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
+      }
+
+      continue;
+    }
+
+    if (isParseBindParam) {
+      return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values");
+    }
+    
     param.schema = pSchema;
-    getSTSRowAppendInfo(schema, pBuilder->rowType, spd, i, &param.toffset, &param.colIdx);
+    getSTSRowAppendInfo(pBuilder->rowType, spd, i, &param.toffset, &param.colIdx);
     CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, &param, &pCxt->msg));
 
     if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
@@ -902,6 +934,8 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
         }
       }
     }
+
+    *gotRow = true;
   }
 
   // *len = pBuilder->extendedRowSize;
@@ -932,20 +966,24 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo
       maxRows = tSize;
     }
 
-    int32_t len = 0;
+    bool gotRow = false;
-    CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &len, tmpTokenBuf));
+    CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, tmpTokenBuf));
-    pDataBlock->size += extendedRowSize;  // len;
+    if (gotRow) {
+      pDataBlock->size += extendedRowSize; //len;
+    }
 
     NEXT_TOKEN(pCxt->pSql, sToken);
     if (TK_NK_RP != sToken.type) {
       return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z);
     }
 
-    (*numOfRows)++;
+    if (gotRow) {
+      (*numOfRows)++;
+    }
   }
 
-  if (0 == (*numOfRows)) {
+  if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
-    return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
+    return  buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -967,7 +1005,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da
   return TSDB_CODE_SUCCESS;
 }
 
-static void destroyCreateSubTbReq(SVCreateTbReq* pReq) {
+void destroyCreateSubTbReq(SVCreateTbReq* pReq) {
   taosMemoryFreeClear(pReq->name);
   taosMemoryFreeClear(pReq->ctbCfg.pTag);
 }
@@ -1001,7 +1039,6 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) {
   taosHashCleanup(pCxt->pVgroupsHashObj);
 
   destroyBlockHashmap(pCxt->pTableBlockHashObj);
-  destroyBlockArrayList(pCxt->pTableDataBlocks);
   destroyBlockArrayList(pCxt->pVgDataBlocks);
 }
 
@@ -1011,23 +1048,41 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) {
 //       VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
 //   [...];
 static int32_t parseInsertBody(SInsertParseContext* pCxt) {
+  int32_t tbNum = 0;
+  
   // for each table
   while (1) {
-    destroyInsertParseContextForTable(pCxt);
-
     SToken sToken;
+    char *tbName = NULL;
+
     // pSql -> tb_name ...
     NEXT_TOKEN(pCxt->pSql, sToken);
 
     // no data in the sql string anymore.
     if (sToken.n == 0) {
-      if (0 == pCxt->totalNum) {
+      if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
         return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");
-        ;
       }
       break;
     }
 
+    if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && tbNum > 0) {
+      return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt");;
+    }
+
+    destroyInsertParseContextForTable(pCxt);
+
+    if (TK_NK_QUESTION == sToken.type) {
+      if (pCxt->pStmtCb) {
+        CHECK_CODE((*pCxt->pStmtCb->getTbNameFn)(pCxt->pStmtCb->pStmt, &tbName));
+        
+        sToken.z = tbName;
+        sToken.n = strlen(tbName);
+      } else {
+        return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z);
+      }
+    }
+    
     SToken tbnameToken = sToken;
     NEXT_TOKEN(pCxt->pSql, sToken);
 
@@ -1053,7 +1108,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
     if (TK_VALUES == sToken.type) {
       // pSql -> (field1_value, ...) [(field1_value2, ...) ...]
       CHECK_CODE(parseValuesClause(pCxt, dataBuf));
-      pCxt->pOutput->insertType = TSDB_QUERY_TYPE_INSERT;
+      TSDB_QUERY_SET_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_INSERT);
+
+      tbNum++;
       continue;
     }
 
@@ -1066,14 +1123,32 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
       }
       // todo
       pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT;
+
+      tbNum++;
       continue;
     }
 
     return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
   }
+  
+  if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
+    SParsedDataColInfo *tags = taosMemoryMalloc(sizeof(pCxt->tags));
+    if (NULL == tags) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    memcpy(tags, &pCxt->tags, sizeof(pCxt->tags));
+    (*pCxt->pStmtCb->setBindInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags);
+    memset(&pCxt->tags, 0, sizeof(pCxt->tags));
+
+    (*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj);
+    pCxt->pVgroupsHashObj = NULL;
+    pCxt->pTableBlockHashObj = NULL;
+    
+    return TSDB_CODE_SUCCESS;
+  }
+  
   // merge according to vgId
-  if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) &&
+  if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
-      taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
     CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks));
   }
   return buildOutput(pCxt);
@@ -1087,29 +1162,43 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
 //   [...];
 int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
   SInsertParseContext context = {
-      .pComCxt = pContext,
+    .pComCxt = pContext,
-      .pSql = (char*)pContext->pSql,
+    .pSql = (char*) pContext->pSql,
-      .msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
+    .msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
-      .pTableMeta = NULL,
+    .pTableMeta = NULL,
-      .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false),
+    .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false),
-      .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false),
+    .totalNum = 0,
-      .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false),
+    .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT),
-      .totalNum = 0,
+    .pStmtCb = pContext->pStmtCb
-      .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT)};
+  };
 
-  if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj ||
+  if (pContext->pStmtCb && *pQuery) {
-      NULL == context.pOutput) {
+    (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, &context.pTableBlockHashObj);
+  } else {
+    context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+    context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
+  }
+  
+  if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj ||
+      NULL == context.pSubTableHashObj || NULL == context.pOutput) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  *pQuery = taosMemoryCalloc(1, sizeof(SQuery));
+  if (pContext->pStmtCb) {
-  if (NULL == *pQuery) {
+    TSDB_QUERY_SET_TYPE(context.pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT);
-    return TSDB_CODE_OUT_OF_MEMORY;
   }
-  (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE;
+
-  (*pQuery)->haveResultSet = false;
+  if (NULL == *pQuery) {
-  (*pQuery)->msgType = TDMT_VND_SUBMIT;
+    *pQuery = taosMemoryCalloc(1, sizeof(SQuery));
-  (*pQuery)->pRoot = (SNode*)context.pOutput;
+    if (NULL == *pQuery) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE;
+    (*pQuery)->haveResultSet = false;
+    (*pQuery)->msgType = TDMT_VND_SUBMIT;
+    (*pQuery)->pRoot = (SNode*)context.pOutput;
+  }
+  
   context.pOutput->payloadType = PAYLOAD_TYPE_KV;
 
   int32_t code = skipInsertInto(&context);
@@ -1119,3 +1208,318 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
   destroyInsertParseContext(&context);
   return code;
 }
+
+
+int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen) {
+  SMsgBuf msg = {.buf = msgBuf, .len =msgBufLen};
+  SToken sToken;
+  int32_t code = 0;
+  char *tbName = NULL;
+  
+  NEXT_TOKEN(pTableName, sToken);
+  
+  if (sToken.n == 0) {
+    return buildInvalidOperationMsg(&msg, "empty table name");
+  }
+
+  code = createSName(pName, &sToken, acctId, dbName, &msg);
+  if (code) {
+    return code;
+  }
+
+  NEXT_TOKEN(pTableName, sToken);
+
+  if (sToken.n > 0) {
+    return buildInvalidOperationMsg(&msg, "table name format is wrong");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) {
+  SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot;
+  int32_t code = 0;
+  SInsertParseContext insertCtx = {
+    .pVgroupsHashObj = pVgHash,
+    .pTableBlockHashObj = pBlockHash,
+    .pOutput = (SVnodeModifOpStmt*)pQuery->pRoot,
+  };
+  
+  // merge according to vgId
+  if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) {
+    CHECK_CODE(mergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks));
+  }
+
+  CHECK_CODE(buildOutput(&insertCtx));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen){
+  STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock;
+  SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; 
+  SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags;
+  if (NULL == tags) {
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  SKVRowBuilder tagBuilder;
+  if (tdInitKVRowBuilder(&tagBuilder) < 0) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
+  SKvParam param = {.builder = &tagBuilder};
+
+  for (int c = 0; c < tags->numOfBound; ++c) {
+    if (bind[c].is_null && bind[c].is_null[0]) {
+      KvRowAppend(&pBuf, NULL, 0, &param);
+      continue;
+    }
+    
+    SSchema* pTagSchema = &pSchema[tags->boundColumns[c] - 1]; // colId starts with 1
+    param.schema = pTagSchema;
+
+    int32_t colLen = pTagSchema->bytes;
+    if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
+      colLen = bind[c].length[0];
+    }
+    
+    CHECK_CODE(KvRowAppend(&pBuf, (char *)bind[c].buffer, colLen, &param));
+  }
+
+  SKVRow row = tdGetKVRowFromBuilder(&tagBuilder);
+  if (NULL == row) {
+    tdDestroyKVRowBuilder(&tagBuilder);
+    return buildInvalidOperationMsg(&pBuf, "tag value expected");
+  }
+  tdSortKVRowByColIdx(row);
+
+  SVCreateTbReq tbReq = {0};
+  CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid));
+  CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq));
+
+  destroyCreateSubTbReq(&tbReq);
+  tdDestroyKVRowBuilder(&tagBuilder);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen) {
+  STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock;
+  SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta);
+  int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
+  SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo;
+  SRowBuilder*        pBuilder = &pDataBlock->rowBuilder;
+  SMemParam param = {.rb = pBuilder};
+  SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; 
+  int32_t rowNum = bind->num;
+  
+  CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
+
+  CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num));
+  
+  for (int32_t r = 0; r < bind->num; ++r) {
+    STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size);  // skip the SSubmitBlk header
+    tdSRowResetBuf(pBuilder, row);
+    
+    // 1. set the parsed value from sql string
+    for (int c = 0; c < spd->numOfBound; ++c) {
+      SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1];
+
+      if (bind[c].buffer_type != pColSchema->type) {
+        return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
+      }
+
+      if (bind[c].num != rowNum) {
+        return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
+      }
+      
+      param.schema = pColSchema;
+      getSTSRowAppendInfo(pBuilder->rowType, spd, c, &param.toffset, &param.colIdx);
+
+      if (bind[c].is_null && bind[c].is_null[r]) {
+        if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+          return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL");
+        }
+        
+        CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, &param));
+      } else {
+        int32_t colLen = pColSchema->bytes;
+        if (IS_VAR_DATA_TYPE(pColSchema->type)) {
+          colLen = bind[c].length[r];
+        }
+        
+        CHECK_CODE(MemRowAppend(&pBuf, (char *)bind[c].buffer + bind[c].buffer_length * r, colLen, &param));
+      }
+    
+      if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) {
+        TSKEY tsKey = TD_ROW_KEY(row);
+        checkTimestamp(pDataBlock, (const char *)&tsKey);
+      }
+    }
+    
+    // set the null value for the columns that do not assign values
+    if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
+      for (int32_t i = 0; i < spd->numOfCols; ++i) {
+        if (spd->cols[i].valStat == VAL_STAT_NONE) {  // the primary TS key is not VAL_STAT_NONE
+          tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i,
+                                spd->cols[i].toffset);
+        }
+      }
+    }
+    
+    pDataBlock->size += extendedRowSize;
+  }
+
+  SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData);
+  if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) {
+    return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) {
+  STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock;
+  SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta);
+  int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
+  SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo;
+  SRowBuilder*        pBuilder = &pDataBlock->rowBuilder;
+  SMemParam param = {.rb = pBuilder};
+  SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; 
+  bool rowStart = (0 == colIdx);
+  bool rowEnd = ((colIdx + 1) == spd->numOfBound);
+
+  if (rowStart) {
+    CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
+    CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num));
+  }
+  
+  for (int32_t r = 0; r < bind->num; ++r) {
+    STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r);  // skip the SSubmitBlk header
+    if (rowStart) {
+      tdSRowResetBuf(pBuilder, row);
+    } else {
+      tdSRowGetBuf(pBuilder, row);
+    }
+    
+    SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx] - 1];
+
+    if (bind->buffer_type != pColSchema->type) {
+      return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
+    }
+
+    if (bind->num != rowNum) {
+      return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
+    }
+    
+    param.schema = pColSchema;
+    getSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, &param.toffset, &param.colIdx);
+
+    if (bind->is_null && bind->is_null[r]) {
+      if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL");
+      }
+      
+      CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, &param));
+    } else {
+      int32_t colLen = pColSchema->bytes;
+      if (IS_VAR_DATA_TYPE(pColSchema->type)) {
+        colLen = bind->length[r];
+      }
+      
+      CHECK_CODE(MemRowAppend(&pBuf, (char *)bind->buffer + bind->buffer_length * r, colLen, &param));
+    }
+  
+    if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) {
+      TSKEY tsKey = TD_ROW_KEY(row);
+      checkTimestamp(pDataBlock, (const char *)&tsKey);
+    }
+    
+    // set the null value for the columns that do not assign values
+    if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
+      for (int32_t i = 0; i < spd->numOfCols; ++i) {
+        if (spd->cols[i].valStat == VAL_STAT_NONE) {  // the primary TS key is not VAL_STAT_NONE
+          tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i,
+                                spd->cols[i].toffset);
+        }
+      }
+    }    
+  }
+
+  if (rowEnd) {
+    pDataBlock->size += extendedRowSize * bind->num;
+
+    SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData);
+    if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) {
+      return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767");
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) {
+  if (fields) {
+    *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD));
+    if (NULL == *fields) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    for (int32_t i = 0; i < boundInfo->numOfBound; ++i) {
+      SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1];
+      strcpy((*fields)[i].name, pTagSchema->name);
+      (*fields)[i].type = pTagSchema->type;
+      (*fields)[i].bytes = pTagSchema->bytes;
+    }
+  }
+
+  *fieldNum = boundInfo->numOfBound;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields) {
+  STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock;
+  SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags;
+  if (NULL == tags) {
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+  
+  SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);  
+  if (tags->numOfBound <= 0) {
+    *fieldNum = 0;
+    *fields = NULL;
+
+    return TSDB_CODE_SUCCESS;
+  }
+
+  CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields));
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields) {
+  STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock;
+  SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta);  
+  if (pDataBlock->boundColumnInfo.numOfBound <= 0) {
+    *fieldNum = 0;
+    if (fields) {
+      *fields = NULL;
+    }
+
+    return TSDB_CODE_SUCCESS;
+  }
+
+  CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields));
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+
+

source/libs/parser/src/parInsertData.c

@@ -18,6 +18,7 @@
 #include "catalog.h"
 #include "parUtil.h"
 #include "querynodes.h"
+#include "parInt.h"
 
 #define IS_RAW_PAYLOAD(t) \
   (((int)(t)) == PAYLOAD_TYPE_RAW)  // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
@@ -102,7 +103,13 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) {
   }
 }
 
-void destroyBoundColumnInfo(SParsedDataColInfo* pColList) {
+void destroyBoundColumnInfo(void* pBoundInfo) {
+  if (NULL == pBoundInfo) {
+    return;
+  }
+
+  SParsedDataColInfo* pColList = (SParsedDataColInfo*)pBoundInfo;
+  
   taosMemoryFreeClear(pColList->boundColumns);
   taosMemoryFreeClear(pColList->cols);
   taosMemoryFreeClear(pColList->colIdxInfo);
@@ -149,7 +156,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) {
+int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) {
   int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq);
   if (pBlocks->nAllocSize - pBlocks->size < len) {
     pBlocks->nAllocSize += len + pBlocks->rowSize;
@@ -506,6 +513,28 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize) {
+  size_t    remain = pDataBlock->nAllocSize - pDataBlock->size;
+  uint32_t nAllocSizeOld = pDataBlock->nAllocSize;
+  
+  // expand the allocated size
+  if (remain < allSize) {
+    pDataBlock->nAllocSize = (pDataBlock->size + allSize) * 1.5;
+
+    char *tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize);
+    if (tmp != NULL) {
+      pDataBlock->pData = tmp;
+      memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size);
+    } else {
+      // do nothing, if allocate more memory failed
+      pDataBlock->nAllocSize = nAllocSizeOld;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) {
   size_t    remain = pDataBlock->nAllocSize - pDataBlock->size;
   const int factor = 5;
@@ -541,3 +570,84 @@ int  initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo
                         pColInfo->boundNullLen);
   return TSDB_CODE_SUCCESS;
 }
+
+
+int32_t qResetStmtDataBlock(void* block, bool keepBuf) {
+  STableDataBlocks* pBlock = (STableDataBlocks*)block;
+
+  if (keepBuf) {
+    taosMemoryFreeClear(pBlock->pData);
+    pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE);
+    if (NULL == pBlock->pData) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    memset(pBlock->pData, 0, sizeof(SSubmitBlk));
+  } else {
+    pBlock->pData  = NULL;
+  }
+  
+  pBlock->ordered  = true;
+  pBlock->prevTS   = INT64_MIN;
+  pBlock->size     = sizeof(SSubmitBlk);
+  pBlock->tsSource = -1;
+  pBlock->numOfTables = 1;
+  pBlock->nAllocSize = TSDB_PAYLOAD_SIZE;
+  pBlock->headerSize = pBlock->size;
+  
+  memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) {
+  *pDst = taosMemoryMalloc(sizeof(STableDataBlocks));
+  if (NULL == *pDst) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  
+  memcpy(*pDst, pSrc, sizeof(STableDataBlocks));
+  ((STableDataBlocks*)(*pDst))->cloned = true;
+  
+  return qResetStmtDataBlock(*pDst, false);
+}
+
+int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc) {
+  int32_t code = qCloneStmtDataBlock(pDst, pSrc);
+  if (code) {
+    return code;
+  }
+
+  STableDataBlocks *pBlock = (STableDataBlocks*)*pDst;
+  pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize);
+  if (NULL == pBlock->pData) {
+    qFreeStmtDataBlock(pBlock);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  memset(pBlock->pData, 0, sizeof(SSubmitBlk));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+void qFreeStmtDataBlock(void* pDataBlock) {
+  if (pDataBlock == NULL) {
+    return;
+  }
+
+  taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData);
+  taosMemoryFreeClear(pDataBlock);
+}
+
+void qDestroyStmtDataBlock(void* pBlock) {
+  if (pBlock == NULL) {
+    return;
+  }
+
+  STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock;
+
+  pDataBlock->cloned = false;
+  destroyDataBlock(pDataBlock);
+}
+

source/libs/parser/src/parTokenizer.c

@@ -52,6 +52,7 @@ static SKeyword keywordTable[] = {
     {"CACHE",         TK_CACHE},
     {"CACHELAST",     TK_CACHELAST},
     {"CAST",          TK_CAST},
+    {"CLUSTER",       TK_CLUSTER},
     {"COLUMN",        TK_COLUMN},
     {"COMMENT",       TK_COMMENT},
     {"COMP",          TK_COMP},
@@ -248,7 +249,6 @@ static SKeyword keywordTable[] = {
     // {"BEFORE",       TK_BEFORE},
     // {"BEGIN",        TK_BEGIN},
     // {"CASCADE",      TK_CASCADE},
-    // {"CLUSTER",      TK_CLUSTER},
     // {"CONFLICT",     TK_CONFLICT},
     // {"COPY",         TK_COPY},
     // {"DEFERRED",     TK_DEFERRED},

source/libs/parser/src/parTranslater.c

@@ -230,6 +230,21 @@ static int32_t initTranslateContext(SParseContext* pParseCxt, STranslateContext*
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t resetTranslateNamespace(STranslateContext* pCxt) {
+  if (NULL != pCxt->pNsLevel) {
+    size_t size = taosArrayGetSize(pCxt->pNsLevel);
+    for (size_t i = 0; i < size; ++i) {
+      taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i));
+    }
+    taosArrayDestroy(pCxt->pNsLevel);
+  }
+  pCxt->pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+  if (NULL == pCxt->pNsLevel) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static void destroyTranslateContext(STranslateContext* pCxt) {
   if (NULL != pCxt->pNsLevel) {
     size_t size = taosArrayGetSize(pCxt->pNsLevel);
@@ -261,9 +276,11 @@ static bool belongTable(const char* currentDb, const SColumnNode* pCol, const ST
   return (0 == cmp);
 }
 
-static SNodeList* getProjectList(SNode* pNode) {
+static SNodeList* getProjectList(const SNode* pNode) {
   if (QUERY_NODE_SELECT_STMT == nodeType(pNode)) {
     return ((SSelectStmt*)pNode)->pProjectionList;
+  } else if (QUERY_NODE_SET_OPERATOR == nodeType(pNode)) {
+    return ((SSetOperator*)pNode)->pProjectionList;
   }
   return NULL;
 }
@@ -543,14 +560,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
       pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
     }
   } else if (nodesIsComparisonOp(pOp)) {
-    if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
+    if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
         TSDB_DATA_TYPE_BLOB == rdt.type) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
     }
     pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
     pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
-  } else {
+  } else if (nodesIsJsonOp(pOp)){
-    // todo json operator
+    if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
+      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
+    }
+    pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
+    pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
   }
   return DEAL_RES_CONTINUE;
 }
@@ -1349,6 +1370,86 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
   return code;
 }
 
+static SNode* createSetOperProject(SNode* pNode) {
+  SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
+  if (NULL == pCol) {
+    return NULL;
+  }
+  pCol->node.resType = ((SExprNode*)pNode)->resType;
+  strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName);
+  strcpy(pCol->node.aliasName, pCol->colName);
+  return (SNode*)pCol;
+}
+
+static bool dataTypeEqual(const SDataType* l, const SDataType* r) {
+  return (l->type == r->type && l->bytes == l->bytes && l->precision == r->precision && l->scale == l->scale);
+}
+
+static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) {
+  SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION);
+  if (NULL == pFunc) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  strcpy(pFunc->functionName, "cast");
+  pFunc->node.resType = dt;
+  if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pFunc->pParameterList, pExpr)) {
+    nodesDestroyNode(pFunc);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) {
+    nodesClearList(pFunc->pParameterList);
+    pFunc->pParameterList = NULL;
+    nodesDestroyNode(pFunc);
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pExpr)->aliasName);
+  }
+  *pCast = (SNode*)pFunc;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* pSetOperator) {
+  SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft);
+  SNodeList* pRightProjections = getProjectList(pSetOperator->pRight);
+  if (LIST_LENGTH(pLeftProjections) != LIST_LENGTH(pRightProjections)) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCORRECT_NUM_OF_COL);
+  }
+
+  SNode* pLeft = NULL;
+  SNode* pRight = NULL;
+  FORBOTH(pLeft, pLeftProjections, pRight, pRightProjections) {
+    SExprNode* pLeftExpr = (SExprNode*)pLeft;
+    SExprNode* pRightExpr = (SExprNode*)pRight;
+    if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) {
+      SNode* pRightFunc = NULL;
+      int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc);
+      if (TSDB_CODE_SUCCESS != code) {
+        return code;
+      }
+      REPLACE_LIST2_NODE(pRightFunc);
+      pRightExpr = (SExprNode*)pRightFunc;
+    }
+    strcpy(pRightExpr->aliasName, pLeftExpr->aliasName);
+    pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0';
+    if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pLeft))) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetOperator) {
+  int32_t code = translateQuery(pCxt, pSetOperator->pLeft);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = resetTranslateNamespace(pCxt);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateQuery(pCxt, pSetOperator->pRight);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateSetOperatorImpl(pCxt, pSetOperator);
+  }
+  return code;
+}
+
 static int64_t getUnitPerMinute(uint8_t precision) {
   switch (precision) {
     case TSDB_TIME_PRECISION_MILLI:
@@ -1797,6 +1898,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+  SNode* pNode;
+  FOREACH(pNode, pStmt->pTags) {
+    SColumnDefNode* pCol = (SColumnDefNode*)pNode;
+    if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){
+      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
   if (NULL == pFuncs) {
     return TSDB_CODE_SUCCESS;
@@ -1832,6 +1944,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
   if (TSDB_CODE_SUCCESS == code) {
     code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = checkTableTags(pCxt, pStmt);
+  }
   return code;
 }
 
@@ -2590,12 +2705,64 @@ static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pSt
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t readFromFile(char* pName, int32_t *len, char **buf) {
+  int64_t filesize = 0;
+  if (taosStatFile(pName, &filesize, NULL) < 0) {
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  *len = filesize;
+
+  if (*len <= 0) {
+    return TSDB_CODE_TSC_FILE_EMPTY;
+  }
+
+  *buf = taosMemoryCalloc(1, *len);
+  if (*buf == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  TdFilePtr tfile = taosOpenFile(pName, O_RDONLY | O_BINARY);
+  if (NULL == tfile) {
+    taosMemoryFreeClear(*buf);
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  int64_t s = taosReadFile(tfile, *buf, *len);
+  if (s != *len) {
+    taosCloseFile(&tfile);
+    taosMemoryFreeClear(*buf);
+    return TSDB_CODE_TSC_APP_ERROR;
+  }
+  taosCloseFile(&tfile);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionStmt* pStmt) {
+  SCreateFuncReq req = {0};
+  strcpy(req.name, pStmt->funcName);
+  req.igExists = pStmt->ignoreExists;
+  req.funcType = pStmt->isAgg ? TSDB_FUNC_TYPE_AGGREGATE : TSDB_FUNC_TYPE_SCALAR;
+  req.scriptType = TSDB_FUNC_SCRIPT_BIN_LIB;
+  req.outputType = pStmt->outputDt.type;
+  req.outputLen = pStmt->outputDt.bytes;
+  req.bufSize = pStmt->bufSize;
+  int32_t code = readFromFile(pStmt->libraryPath, &req.codeLen, &req.pCode);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildCmdMsg(pCxt, TDMT_MND_CREATE_FUNC, (FSerializeFunc)tSerializeSCreateFuncReq, &req);
+  }
+  return code;
+}
+
 static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
   int32_t code = TSDB_CODE_SUCCESS;
   switch (nodeType(pNode)) {
     case QUERY_NODE_SELECT_STMT:
       code = translateSelect(pCxt, (SSelectStmt*)pNode);
       break;
+    case QUERY_NODE_SET_OPERATOR:
+      code = translateSetOperator(pCxt, (SSetOperator*)pNode);
+      break;
     case QUERY_NODE_CREATE_DATABASE_STMT:
       code = translateCreateDatabase(pCxt, (SCreateDatabaseStmt*)pNode);
       break;
@@ -2688,6 +2855,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
     case QUERY_NODE_DROP_STREAM_STMT:
       code = translateDropStream(pCxt, (SDropStreamStmt*)pNode);
       break;
+    case QUERY_NODE_CREATE_FUNCTION_STMT:
+      code = translateCreateFunction(pCxt, (SCreateFunctionStmt*)pNode);
+      break;
     default:
       break;
   }
@@ -2705,8 +2875,8 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) {
   return code;
 }
 
-static int32_t extractSelectResultSchema(const SSelectStmt* pSelect, int32_t* numOfCols, SSchema** pSchema) {
+static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* numOfCols, SSchema** pSchema) {
-  *numOfCols = LIST_LENGTH(pSelect->pProjectionList);
+  *numOfCols = LIST_LENGTH(pProjections);
   *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema));
   if (NULL == (*pSchema)) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -2714,7 +2884,7 @@ static int32_t extractSelectResultSchema(const SSelectStmt* pSelect, int32_t* nu
 
   SNode*  pNode;
   int32_t index = 0;
-  FOREACH(pNode, pSelect->pProjectionList) {
+  FOREACH(pNode, pProjections) {
     SExprNode* pExpr = (SExprNode*)pNode;
     (*pSchema)[index].type = pExpr->resType.type;
     (*pSchema)[index].bytes = pExpr->resType.bytes;
@@ -2773,7 +2943,8 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS
 
   switch (nodeType(pRoot)) {
     case QUERY_NODE_SELECT_STMT:
-      return extractSelectResultSchema((SSelectStmt*)pRoot, numOfCols, pSchema);
+    case QUERY_NODE_SET_OPERATOR:
+      return extractQueryResultSchema(getProjectList(pRoot), numOfCols, pSchema);
     case QUERY_NODE_EXPLAIN_STMT:
       return extractExplainResultSchema(numOfCols, pSchema);
     case QUERY_NODE_DESCRIBE_STMT:
@@ -2802,6 +2973,7 @@ static const char* getSysDbName(ENodeType type) {
     case QUERY_NODE_SHOW_BNODES_STMT:
     case QUERY_NODE_SHOW_SNODES_STMT:
     case QUERY_NODE_SHOW_LICENCE_STMT:
+    case QUERY_NODE_SHOW_CLUSTER_STMT:
       return TSDB_INFORMATION_SCHEMA_DB;
     case QUERY_NODE_SHOW_CONNECTIONS_STMT:
     case QUERY_NODE_SHOW_QUERIES_STMT:
@@ -2844,6 +3016,8 @@ static const char* getSysTableName(ENodeType type) {
       return TSDB_INS_TABLE_SNODES;
     case QUERY_NODE_SHOW_LICENCE_STMT:
       return TSDB_INS_TABLE_LICENCES;
+    case QUERY_NODE_SHOW_CLUSTER_STMT:
+      return TSDB_INS_TABLE_CLUSTER;
     case QUERY_NODE_SHOW_CONNECTIONS_STMT:
       return TSDB_PERFS_TABLE_CONNECTIONS;
     case QUERY_NODE_SHOW_QUERIES_STMT:
@@ -3173,17 +3347,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
 
 static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
                              SKVRowBuilder* pBuilder) {
+  if(pSchema->type == TSDB_DATA_TYPE_JSON){
+    if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
+    }
+
+    return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
+  }
+
   if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
     return pCxt->errCode;
   }
-  SVariant var;
+
-  valueNodeToVariant(pVal, &var);
+  if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
-  char    tagVal[TSDB_MAX_TAGS_LEN] = {0};
+    // todo
-  int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
+  }else{
-  if (TSDB_CODE_SUCCESS == code) {
+    tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
-    tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal);
   }
-  return code;
+
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,
@@ -3360,6 +3542,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
     case QUERY_NODE_SHOW_SNODES_STMT:
     case QUERY_NODE_SHOW_CONNECTIONS_STMT:
     case QUERY_NODE_SHOW_QUERIES_STMT:
+    case QUERY_NODE_SHOW_CLUSTER_STMT:
       code = rewriteShow(pCxt, pQuery);
       break;
     case QUERY_NODE_CREATE_TABLE_STMT:
@@ -3384,6 +3567,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
 static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
   switch (nodeType(pQuery->pRoot)) {
     case QUERY_NODE_SELECT_STMT:
+    case QUERY_NODE_SET_OPERATOR:
     case QUERY_NODE_EXPLAIN_STMT:
       pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
       pQuery->haveResultSet = true;

source/libs/parser/src/parUtil.c

@@ -14,6 +14,7 @@
  */
 
 #include "parUtil.h"
+#include "cJSON.h"
 
 static char* getSyntaxErrFormat(int32_t errCode) {
   switch (errCode) {
@@ -115,6 +116,10 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "sliding value no larger than the interval value";
     case TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL:
       return "sliding value can not less than 1% of interval value";
+    case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG:
+      return "Only one tag if there is a json tag";
+    case TSDB_CODE_PAR_INCORRECT_NUM_OF_COL:
+      return "Query block has incorrect number of result columns";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:
@@ -215,24 +220,178 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) {
 }
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
-  // delete escape character: \\, \', \"
+  if (len <=0 || dlen <= 0) return 0;
+
   char delim = src[0];
-  int32_t cnt = 0;
   int32_t j = 0;
   for (uint32_t k = 1; k < len - 1; ++k) {
     if (j >= dlen) {
-      break;
+      dst[j - 1] = '\0';
+      return j;
     }
-    if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) {
+    if (src[k] == delim && src[k + 1] == delim) {   // deal with "", ''
       dst[j] = src[k + 1];
-      cnt++;
       j++;
       k++;
       continue;
     }
+
+    if (src[k] == '\\') {   // deal with escape character
+      if(src[k+1] == 'n'){
+        dst[j] = '\n';
+      }else if(src[k+1] == 'r'){
+        dst[j] = '\r';
+      }else if(src[k+1] == 't'){
+        dst[j] = '\t';
+      }else if(src[k+1] == '\\'){
+        dst[j] = '\\';
+      }else if(src[k+1] == '\''){
+        dst[j] = '\'';
+      }else if(src[k+1] == '"'){
+        dst[j] = '"';
+      }else if(src[k+1] == '%' || src[k+1] == '_'){
+        dst[j++] = src[k];
+        dst[j] = src[k+1];
+      }else{
+        dst[j] = src[k+1];
+      }
+      j++;
+      k++;
+      continue;
+    }
+
     dst[j] = src[k];
     j++;
   }
   dst[j] = '\0';
   return j;
 }
+
+static bool isValidateTag(char *input) {
+  if (!input) return false;
+  for (size_t i = 0; i < strlen(input); ++i) {
+    if (isprint(input[i]) == 0) return false;
+  }
+  return true;
+}
+
+int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
+  // set json NULL data
+  uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
+  int jsonIndex = startColId + 1;
+  if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
+    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // set json real data
+  cJSON *root = cJSON_Parse(json);
+  if (root == NULL){
+    return buildSyntaxErrMsg(pMsgBuf, "json parse error", json);
+  }
+
+  int size = cJSON_GetArraySize(root);
+  if(!cJSON_IsObject(root)){
+    return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json);
+  }
+
+  int retCode = 0;
+  char *tagKV = NULL;
+  SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
+  for(int i = 0; i < size; i++) {
+    cJSON* item = cJSON_GetArrayItem(root, i);
+    if (!item) {
+      qError("json inner error:%d", i);
+      retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
+      goto end;
+    }
+
+    char *jsonKey = item->string;
+    if(!isValidateTag(jsonKey)){
+      retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
+      goto end;
+    }
+//    if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){
+//      tscError("json key too long error");
+//      retCode =  tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
+//      goto end;
+//    }
+    size_t keyLen = strlen(jsonKey);
+    if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
+      continue;
+    }
+    // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES
+    tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1);
+    if(!tagKV) {
+      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto end;
+    }
+    strncpy(varDataVal(tagKV), jsonKey, keyLen);
+    varDataSetLen(tagKV, keyLen);
+    if(taosHashGetSize(keyHash) == 0){
+      uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);   // add json type
+    }
+    taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
+
+    if(item->type == cJSON_String){     // add json value  format: type|data
+      char *jsonValue = item->valuestring;
+      int32_t valLen = (int32_t)strlen(jsonValue);
+      int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
+      char *tmp = taosMemoryRealloc(tagKV, totalLen);
+      if(!tmp) {
+        retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+        goto end;
+      }
+      tagKV = tmp;
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = TSDB_DATA_TYPE_NCHAR;
+      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
+                                                  (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
+        qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
+        retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
+        goto end;
+      }
+
+      varDataSetLen(valueData, valLen);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
+    }else if(item->type == cJSON_Number){
+      if(!isfinite(item->valuedouble)){
+        qError("json value is invalidate");
+        retCode =  buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
+        goto end;
+      }
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE;
+      if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble;
+      else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES);
+    }else if(item->type == cJSON_True || item->type == cJSON_False){
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = TSDB_DATA_TYPE_BOOL;
+      *valueData = (char)(item->valueint);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
+    }else if(item->type == cJSON_NULL){
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      *valueType = TSDB_DATA_TYPE_NULL;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+    }
+    else{
+      retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
+      goto end;
+    }
+  }
+
+  if(taosHashGetSize(keyHash) == 0){  // set json NULL true
+    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
+  }
+
+end:
+  taosMemoryFree(tagKV);
+  taosHashCleanup(keyHash);
+  cJSON_Delete(root);
+  return retCode;
+}