Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/vnode_refact1

example/src/tmq.c

@@ -22,6 +22,7 @@
 static int  running = 1;
 static void msg_process(TAOS_RES* msg) {
   char buf[1024];
+  memset(buf, 0, 1024);
   printf("topic: %s\n", tmq_get_topic_name(msg));
   printf("vg:%d\n", tmq_get_vgroup_id(msg));
   while (1) {
@@ -220,7 +221,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
       msg_process(tmqmessage);
       tmq_message_destroy(tmqmessage);
 
-      if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
+      /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/
     }
   }
 

include/common/tmsg.h

@@ -324,7 +324,7 @@ typedef struct SEpSet {
 int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp);
 int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
 int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
-void*   taosDecodeSEpSet(void* buf, SEpSet* pEp);
+void*   taosDecodeSEpSet(const void* buf, SEpSet* pEp);
 
 typedef struct {
   int8_t  connType;
@@ -1291,10 +1291,14 @@ typedef struct {
 int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
 int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
 
+typedef struct {
+  int64_t consumerId;
+} SMqConsumerLostMsg;
+
 typedef struct {
   int32_t topicNum;
   int64_t consumerId;
-  char*   consumerGroup;
+  char    cgroup[TSDB_CGROUP_LEN];
   SArray* topicNames;  // SArray<char*>
 } SCMSubscribeReq;
 
@@ -1302,7 +1306,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pReq->topicNum);
   tlen += taosEncodeFixedI64(buf, pReq->consumerId);
-  tlen += taosEncodeString(buf, pReq->consumerGroup);
+  tlen += taosEncodeString(buf, pReq->cgroup);
 
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i));
@@ -1313,7 +1317,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
 static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
   buf = taosDecodeFixedI32(buf, &pReq->topicNum);
   buf = taosDecodeFixedI64(buf, &pReq->consumerId);
-  buf = taosDecodeString(buf, &pReq->consumerGroup);
+  buf = taosDecodeStringTo(buf, pReq->cgroup);
   pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*));
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     char* name;
@@ -1389,7 +1393,7 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq
 }
 
 typedef struct {
-  const char* key;
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
   SArray* lostConsumers;     // SArray<int64_t>
   SArray* removedConsumers;  // SArray<int64_t>
   SArray* newConsumers;      // SArray<int64_t>
@@ -1400,7 +1404,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) {
   if (pRebSub == NULL) {
     goto _err;
   }
-  pRebSub->key = strdup(key);
+  strcpy(pRebSub->key, key);
   pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t));
   if (pRebSub->lostConsumers == NULL) {
     goto _err;
@@ -1425,6 +1429,7 @@ _err:
 // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or
 // deserialization
 typedef struct {
+  int8_t*   mqInReb;
   SHashObj* rebSubHash;  // SHashObj<key, SMqRebSubscribe>
 } SMqDoRebalanceMsg;
 
@@ -1936,6 +1941,40 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
   return buf;
 }
 
+typedef struct {
+  int64_t leftForVer;
+  int32_t vgId;
+  int64_t oldConsumerId;
+  int64_t newConsumerId;
+  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  char*   qmsg;
+} SMqRebVgReq;
+
+static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
+  tlen += taosEncodeFixedI32(buf, pReq->vgId);
+  tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
+  tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
+  tlen += taosEncodeString(buf, pReq->subKey);
+  tlen += taosEncodeString(buf, pReq->qmsg);
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) {
+  buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
+  buf = taosDecodeFixedI32(buf, &pReq->vgId);
+  buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
+  buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
+  buf = taosDecodeStringTo(buf, pReq->subKey);
+  buf = taosDecodeString(buf, &pReq->qmsg);
+  return (void*)buf;
+}
+
+typedef struct {
+  int8_t reserved;
+} SMqRebVgRsp;
+
 typedef struct {
   int64_t leftForVer;
   int32_t vgId;
@@ -2406,6 +2445,7 @@ typedef struct {
   int64_t consumerId;
 } SMqRspHead;
 
+#if 0
 typedef struct {
   SMsgHead head;
 
@@ -2419,6 +2459,17 @@ typedef struct {
   uint64_t reqId;
   char     topic[TSDB_TOPIC_FNAME_LEN];
 } SMqPollReq;
+#endif
+
+typedef struct {
+  SMsgHead head;
+  char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int32_t  epoch;
+  uint64_t reqId;
+  int64_t  consumerId;
+  int64_t  blockingTime;
+  int64_t  currentOffset;
+} SMqPollReqV2;
 
 typedef struct {
   int32_t vgId;
@@ -2492,13 +2543,81 @@ static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRs
   return (void*)buf;
 }
 
+typedef struct {
+  SMqRspHead head;
+  int64_t    reqOffset;
+  int64_t    rspOffset;
+  int32_t    skipLogNum;
+  int32_t    blockNum;
+  int8_t     withTbName;
+  int8_t     withSchema;
+  int8_t     withTag;
+  int8_t     withTagSchema;
+  SArray*    blockDataLen;    // SArray<int32_t>
+  SArray*    blockData;       // SArray<SRetrieveTableRsp*>
+  SArray*    blockTbName;     // SArray<char*>
+  SArray*    blockSchema;     // SArray<SSchemaWrapper>
+  SArray*    blockTags;       // SArray<kvrow>
+  SArray*    blockTagSchema;  // SArray<kvrow>
+} SMqDataBlkRsp;
+
+static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
+  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
+  tlen += taosEncodeFixedI32(buf, pRsp->blockNum);
+  if (pRsp->blockNum != 0) {
+    tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
+    tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTag);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
+      void*   data = taosArrayGetP(pRsp->blockData, i);
+      tlen += taosEncodeFixedI32(buf, bLen);
+      tlen += taosEncodeBinary(buf, data, bLen);
+    }
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) {
+  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
+  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
+  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
+  buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
+  pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  if (pRsp->blockNum != 0) {
+    buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
+    buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTag);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = 0;
+      void*   data = NULL;
+      buf = taosDecodeFixedI32(buf, &bLen);
+      buf = taosDecodeBinary(buf, &data, bLen);
+      taosArrayPush(pRsp->blockDataLen, &bLen);
+      taosArrayPush(pRsp->blockData, &data);
+    }
+  }
+  return (void*)buf;
+}
+
 typedef struct {
   SMqRspHead head;
   char       cgroup[TSDB_CGROUP_LEN];
   SArray*    topics;  // SArray<SMqSubTopicEp>
 } SMqCMGetSubEpRsp;
 
-static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
+static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
+  // taosMemoryFree(pSubTopicEp->schema.pSchema);
+  taosArrayDestroy(pSubTopicEp->vgs);
+}
 
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
   int32_t tlen = 0;

include/common/tmsgdef.h

@@ -146,6 +146,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
@@ -177,6 +178,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)

include/dnode/mnode/sdb/sdb.h

@@ -92,7 +92,13 @@ extern "C" {
 typedef struct SMnode  SMnode;
 typedef struct SSdbRaw SSdbRaw;
 typedef struct SSdbRow SSdbRow;
-typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType;
+
+typedef enum {
+  SDB_KEY_BINARY = 1,
+  SDB_KEY_INT32 = 2,
+  SDB_KEY_INT64 = 3,
+} EKeyType;
+
 typedef enum {
   SDB_STATUS_INIT = 0,
   SDB_STATUS_CREATING = 1,

include/util/talgo.h

@@ -27,6 +27,11 @@ extern "C" {
 typedef int32_t (*__compar_fn_t)(const void *, const void *);
 #endif
 
+typedef void *(*FCopy)(void *);
+typedef void (*FDelete)(void *);
+typedef int32_t (*FEncode)(void **buf, const void *dst);
+typedef void *(*FDecode)(const void *buf, void *dst);
+
 #define TD_EQ 0x1
 #define TD_GT 0x2
 #define TD_LT 0x4

include/util/taoserror.h

@@ -279,6 +279,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)

include/util/tarray.h

@@ -199,6 +199,13 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize);
  */
 SArray* taosArrayDup(const SArray* pSrc);
 
+/**
+ * deep copy a new array
+ * @param pSrc
+ */
+SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
+
+
 /**
  * clear the array (remove all element)
  * @param pArray
@@ -212,19 +219,9 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
-
-/**
- * destroy array list
- * @param pArray
- */
 void* taosArrayDestroy(SArray* pArray);
-
+void  taosArrayDestroyP(SArray* pArray, FDelete fp);
-/**
+void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
- *
- * @param pArray
- * @param fp
- */
-void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
 
 /**
  * sort the array
@@ -272,6 +269,9 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t
 
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
 
+int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
+void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tlog.h

@@ -60,6 +60,7 @@ extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
 extern int32_t metaDebugFlag;
+extern int32_t fnDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();

source/client/inc/clientInt.h

@@ -188,10 +188,12 @@ typedef struct SRequestSendRecvBody {
 
 typedef struct {
   int8_t         resType;
-  char*   topic;
+  char           topic[TSDB_TOPIC_FNAME_LEN];
-  SArray* res;  // SArray<SReqResultInfo>
-  int32_t resIter;
   int32_t        vgId;
+  SSchemaWrapper schema;
+  int32_t        resIter;
+  SMqDataBlkRsp  rsp;
+  SReqResultInfo resInfo;
 } SMqRspObj;
 
 typedef struct SRequestObj {
@@ -211,16 +213,24 @@ typedef struct SRequestObj {
   SRequestSendRecvBody body;
 } SRequestObj;
 
+void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
+void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
+void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
+void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
+
 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  int32_t    resIter = msg->resIter == -1 ? 0 : msg->resIter;
+  return (SReqResultInfo*)&msg->resInfo;
-  return (SReqResultInfo*)taosArrayGet(msg->res, resIter);
 }
 
-static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) {
+static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  if (++msg->resIter < taosArrayGetSize(msg->res)) {
+  msg->resIter++;
-    return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter);
+  if (msg->resIter < msg->rsp.blockNum) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
+    setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4);
+    return &msg->resInfo;
   }
   return NULL;
 }
@@ -273,12 +283,6 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
-void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
-void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
-void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
-void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
-int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
-
 // --- heartbeat
 // global, called by mgmt
 int  hbMgrInit();

source/client/src/clientMain.c

@@ -14,12 +14,12 @@
  */
 
 #include "catalog.h"
-#include "scheduler.h"
 #include "clientInt.h"
-#include "clientStmt.h"
 #include "clientLog.h"
+#include "clientStmt.h"
 #include "os.h"
 #include "query.h"
+#include "scheduler.h"
 #include "tglobal.h"
 #include "tmsg.h"
 #include "tref.h"
@@ -178,24 +178,23 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
 
   } else if (TD_RES_TMQ(res)) {
     SMqRspObj      *msg = ((SMqRspObj *)res);
-    if (msg->resIter == -1) msg->resIter++;
+    SReqResultInfo *pResultInfo;
-    SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter);
+    if (msg->resIter == -1) {
+      pResultInfo = tmqGetNextResInfo(res, true);
+    } else {
+      pResultInfo = tmqGetCurResInfo(res);
+    }
     if (pResultInfo->current < pResultInfo->numOfRows) {
       doSetOneRowPtr(pResultInfo);
       pResultInfo->current += 1;
       return pResultInfo->row;
     } else {
-      msg->resIter++;
+      pResultInfo = tmqGetNextResInfo(res, true);
-      if (msg->resIter < taosArrayGetSize(msg->res)) {
+      if (pResultInfo == NULL) return NULL;
-        pResultInfo = taosArrayGet(msg->res, msg->resIter);
       doSetOneRowPtr(pResultInfo);
       pResultInfo->current += 1;
       return pResultInfo->row;
-      } else {
-        return NULL;
     }
-    }
-
   } else {
     // assert to avoid un-initialization error
     ASSERT(0);
@@ -455,7 +454,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
     (*numOfRows) = pResultInfo->numOfRows;
     return pRequest->code;
   } else if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true);
     if (pResultInfo == NULL) return -1;
 
     pResultInfo->current = pResultInfo->numOfRows;
@@ -474,7 +473,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
   }
 
   if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false);
     if (pResultInfo == NULL) {
       (*numOfRows) = 0;
       return 0;
@@ -710,10 +709,8 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
   return stmtBindBatch(stmt, bind);
 }
 
-
 TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
   // TODO
   return NULL;
 }
 
-

source/client/src/tmq.c

@@ -75,18 +75,18 @@ struct tmq_t {
   char   groupId[TSDB_CGROUP_LEN];
   char   clientId[256];
   int8_t autoCommit;
-  int8_t         inWaiting;
+  /*int8_t         inWaiting;*/
   int64_t        consumerId;
   int32_t        epoch;
   int32_t        resetOffsetCfg;
   int64_t        status;
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
-  int32_t        nextTopicIdx;
+  /*int32_t        nextTopicIdx;*/
   int8_t  epStatus;
   int32_t epSkipCnt;
-  int32_t        waitingRequest;
+  /*int32_t        waitingRequest;*/
-  int32_t        readyRequest;
+  /*int32_t        readyRequest;*/
   SArray*     clientTopics;  // SArray<SMqClientTopic>
   STaosQueue* mqueue;        // queue of tmq_message_t
   STaosQall*  qall;
@@ -134,7 +134,7 @@ typedef struct {
   int32_t         epoch;
   SMqClientVg*    vgHandle;
   SMqClientTopic* topicHandle;
-  SMqPollRspV2    msg;
+  SMqDataBlkRsp   msg;
 } SMqPollRspWrapper;
 
 typedef struct {
@@ -145,6 +145,7 @@ typedef struct {
 
 typedef struct {
   tmq_t*  tmq;
+  int32_t code;
   int32_t sync;
   tsem_t  rspSem;
 } SMqAskEpCbParam;
@@ -255,7 +256,12 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
 void tmq_list_destroy(tmq_list_t* list) {
   SArray* container = &list->container;
   /*taosArrayDestroy(container);*/
-  taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree);
+  int32_t sz = taosArrayGetSize(container);
+  for (int32_t i = 0; i < sz; i++) {
+    char* str = taosArrayGetP(container, i);
+    taosMemoryFree(str);
+  }
+  taosArrayDestroy(container);
 }
 
 static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
@@ -322,12 +328,12 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
     return NULL;
   }
   pTmq->pTscObj = (STscObj*)conn;
-  pTmq->inWaiting = 0;
+  /*pTmq->inWaiting = 0;*/
   pTmq->status = 0;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
-  pTmq->waitingRequest = 0;
+  /*pTmq->waitingRequest = 0;*/
-  pTmq->readyRequest = 0;
+  /*pTmq->readyRequest = 0;*/
   pTmq->epStatus = 0;
   pTmq->epSkipCnt = 0;
   // set conf
@@ -367,12 +373,12 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ);
   if (pTmq->pTscObj == NULL) return NULL;
 
-  pTmq->inWaiting = 0;
+  /*pTmq->inWaiting = 0;*/
   pTmq->status = 0;
   pTmq->pollCnt = 0;
   pTmq->epoch = 0;
-  pTmq->waitingRequest = 0;
+  /*pTmq->waitingRequest = 0;*/
-  pTmq->readyRequest = 0;
+  /*pTmq->readyRequest = 0;*/
   pTmq->epStatus = 0;
   pTmq->epSkipCnt = 0;
   // set conf
@@ -496,7 +502,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
   SCMSubscribeReq req;
   req.topicNum = sz;
   req.consumerId = tmq->consumerId;
-  req.consumerGroup = strdup(tmq->groupId);
+  strcpy(req.cgroup, tmq->groupId);
   req.topicNames = taosArrayInit(sz, sizeof(void*));
 
   for (int i = 0; i < sz; i++) {
@@ -857,7 +863,6 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
 #endif
 
 int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
-  /*printf("recv poll\n");*/
   SMqPollCbParam* pParam = (SMqPollCbParam*)param;
   SMqClientVg*    pVg = pParam->pVg;
   SMqClientTopic* pTopic = pParam->pTopic;
@@ -870,17 +875,15 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
-    /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
+    // do not write into queue since updating epoch reset
-    /*tsem_post(&tmq->rspSem);*/
     tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
             tmqEpoch);
+    /*tsem_post(&tmq->rspSem);*/
     return 0;
   }
 
   if (msgEpoch != tmqEpoch) {
     tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
-  } else {
-    atomic_sub_fetch_32(&tmq->waitingRequest, 1);
   }
 
 #if 0
@@ -902,45 +905,33 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 #endif
 
-  /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
-  /*tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));*/
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper));
   if (pRspWrapper == NULL) {
     tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
     goto CREATE_MSG_FAIL;
   }
+
   pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP;
   pRspWrapper->vgHandle = pVg;
   pRspWrapper->topicHandle = pTopic;
-  /*memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));*/
-  memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
-  tDecodeSMqPollRspV2(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
-  // TODO: alloc mem
-  /*pRsp->*/
-  /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
 
-#if 0
+  memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
-  if (pRsp->msg.numOfTopics == 0) {
+
-    /*printf("no data\n");*/
+  tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
-    taosFreeQitem(pRsp);
-    goto CREATE_MSG_FAIL;
-  }
-#endif
 
   tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId,
            pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
-  atomic_add_fetch_32(&tmq->readyRequest, 1);
   /*tsem_post(&tmq->rspSem);*/
-  return 0;
 
+  return 0;
 CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
   /*tsem_post(&tmq->rspSem);*/
-  return code;
+  return -1;
 }
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
@@ -1023,6 +1014,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
 int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
   tmq_t*           tmq = pParam->tmq;
+  pParam->code = code;
   if (code != 0) {
     tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync);
     goto END;
@@ -1062,6 +1054,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
 
     taosWriteQitem(tmq->mqueue, pWrapper);
     /*tsem_post(&tmq->rspSem);*/
+    taosMemoryFree(pParam);
   }
 
 END:
@@ -1073,6 +1066,7 @@ END:
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
+  int32_t code = 0;
   int8_t  epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
   if (epStatus == 1) {
     int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
@@ -1130,8 +1124,12 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
-  if (sync) tsem_wait(&pParam->rspSem);
+  if (sync) {
-  return 0;
+    tsem_wait(&pParam->rspSem);
+    code = pParam->code;
+    taosMemoryFree(pParam);
+  }
+  return code;
 }
 
 tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
@@ -1157,7 +1155,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
   return TMQ_RESP_ERR__FAIL;
 }
 
-SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
   int64_t reqOffset;
   if (pVg->currentOffset >= 0) {
     reqOffset = pVg->currentOffset;
@@ -1169,13 +1167,18 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
     reqOffset = tmq->resetOffsetCfg;
   }
 
-  SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq));
+  SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2));
   if (pReq == NULL) {
     return NULL;
   }
 
-  strcpy(pReq->topic, pTopic->topicName);
+  /*strcpy(pReq->topic, pTopic->topicName);*/
-  strcpy(pReq->cgroup, tmq->groupId);
+  /*strcpy(pReq->cgroup, tmq->groupId);*/
+
+  int32_t tlen = strlen(tmq->groupId);
+  memcpy(pReq->subKey, tmq->groupId, tlen);
+  pReq->subKey[tlen] = TMQ_SEPARATOR;
+  strcpy(pReq->subKey + tlen + 1, pTopic->topicName);
 
   pReq->blockingTime = blockingTime;
   pReq->consumerId = tmq->consumerId;
@@ -1184,102 +1187,27 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
   pReq->reqId = generateRequestId();
 
   pReq->head.vgId = htonl(pVg->vgId);
-  pReq->head.contLen = htonl(sizeof(SMqPollReq));
+  pReq->head.contLen = htonl(sizeof(SMqPollReqV2));
   return pReq;
 }
 
 SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
   pRspObj->resType = RES_TYPE__TMQ;
-  pRspObj->topic = strdup(pWrapper->topicHandle->topicName);
+  strncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
-  pRspObj->resIter = -1;
   pRspObj->vgId = pWrapper->vgHandle->vgId;
-  SMqPollRspV2* pRsp = &pWrapper->msg;
+  pRspObj->resIter = -1;
-  int32_t       blockNum = taosArrayGetSize(pRsp->blockPos);
+  memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp));
-  pRspObj->res = taosArrayInit(blockNum, sizeof(SReqResultInfo));
+
-  for (int32_t i = 0; i < blockNum; i++) {
+  /*SRetrieveTableRsp* pRetrieve = taosArrayGetP(pWrapper->msg.blockData, 0);*/
-    int32_t            pos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
+  pRspObj->resInfo.totalRows = 0;
-    SRetrieveTableRsp* pRetrieve = POINTER_SHIFT(pRsp->blockData, pos);
+  pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-    SReqResultInfo     resInfo = {0};
+  setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
-    resInfo.totalRows = 0;
+
-    resInfo.precision = TSDB_TIME_PRECISION_MILLI;
+  taosFreeQitem(pWrapper);
-    setResSchemaInfo(&resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
-    setQueryResultFromRsp(&resInfo, pRetrieve, true);
-    taosArrayPush(pRspObj->res, &resInfo);
-  }
   return pRspObj;
 }
 
-#if 0
-tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
-  tmq_message_t* msg = NULL;
-  for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
-    SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
-      SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-      int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
-      /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
-      /*continue;*/
-      /*}*/
-      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
-      if (pReq == NULL) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
-        return NULL;
-      }
-
-      SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
-      if (pParam == NULL) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        // TODO: out of mem
-        return NULL;
-      }
-      pParam->tmq = tmq;
-      pParam->pVg = pVg;
-      pParam->epoch = tmq->epoch;
-      pParam->sync = 1;
-      pParam->msg = &msg;
-      tsem_init(&pParam->rspSem, 0, 0);
-
-      SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
-      if (sendInfo == NULL) {
-        return NULL;
-      }
-
-      sendInfo->msgInfo = (SDataBuf){
-          .pData = pReq,
-          .len = sizeof(SMqPollReq),
-          .handle = NULL,
-      };
-      sendInfo->requestId = generateRequestId();
-      sendInfo->requestObjRefId = 0;
-      sendInfo->param = pParam;
-      sendInfo->fp = tmqPollCb;
-      sendInfo->msgType = TDMT_VND_CONSUME;
-
-      int64_t transporterId = 0;
-      /*printf("send poll\n");*/
-      atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
-      pVg->pollCnt++;
-      tmq->pollCnt++;
-
-      tsem_wait(&pParam->rspSem);
-      tmq_message_t* nmsg = NULL;
-      while (1) {
-        taosReadQitem(tmq->mqueue, (void**)&nmsg);
-        if (nmsg == NULL) continue;
-        while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {
-          taosReadQitem(tmq->mqueue, (void**)&nmsg);
-        }
-        return nmsg;
-      }
-    }
-  }
-  return NULL;
-}
-#endif
-
 int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
   /*printf("call poll\n");*/
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
@@ -1301,7 +1229,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 #endif
       }
       atomic_store_32(&pVg->vgSkipCnt, 0);
-      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         /*tsem_post(&tmq->rspSem);*/
@@ -1332,7 +1260,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       sendInfo->msgInfo = (SDataBuf){
           .pData = pReq,
-          .len = sizeof(SMqPollReq),
+          .len = sizeof(SMqPollReqV2),
           .handle = NULL,
       };
       sendInfo->requestId = pReq->reqId;
@@ -1343,7 +1271,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
-      atomic_add_fetch_32(&tmq->waitingRequest, 1);
+      /*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/
       tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
                pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
@@ -1385,7 +1313,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
 
     if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
-      atomic_sub_fetch_32(&tmq->readyRequest, 1);
+      /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
       /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
       if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) {
         /*printf("epoch match\n");*/
@@ -1393,7 +1321,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
         pVg->currentOffset = pollRspWrapper->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        if (pollRspWrapper->msg.dataLen == 0) {
+        if (pollRspWrapper->msg.blockNum == 0) {
           taosFreeQitem(pollRspWrapper);
           rspWrapper = NULL;
           continue;
@@ -1449,7 +1377,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
   // TODO: put into another thread or delayed queue
   int64_t status = atomic_load_64(&tmq->status);
-  tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
+  while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) {
+    tscDebug("not ready, retry\n");
+    taosSsleep(1);
+  }
 
   rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
   if (rspObj) {

source/common/src/tglobal.c

@@ -289,6 +289,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1;
   return 0;
 }
 
@@ -473,6 +474,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
   tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
   tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
   fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
+  fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32;
 }
 
 static int32_t taosSetClientCfg(SConfig *pCfg) {

source/common/src/tmsg.c

@@ -125,14 +125,14 @@ int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   return tlen;
 }
 
-void *taosDecodeSEpSet(void *buf, SEpSet *pEp) {
+void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) {
   buf = taosDecodeFixedI8(buf, &pEp->inUse);
   buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
     buf = taosDecodeFixedU16(buf, &pEp->eps[i].port);
     buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn);
   }
-  return buf;
+  return (void *)buf;
 }
 
 static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) {
@@ -3643,8 +3643,6 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
 

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -211,6 +211,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -250,7 +250,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -263,10 +263,11 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
+  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);  
   dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, DEFAULT_HANDLE);

source/dnode/mnode/impl/inc/mndConsumer.h

@@ -22,16 +22,17 @@
 extern "C" {
 #endif
 
-
 enum {
-  MQ_CONSUMER_STATUS__INIT = 1,
+  // MQ_CONSUMER_STATUS__INIT = 1,
-  MQ_CONSUMER_STATUS__IDLE,
+  MQ_CONSUMER_STATUS__MODIFY = 1,
-  MQ_CONSUMER_STATUS__ACTIVE,
+  MQ_CONSUMER_STATUS__MODIFY_IN_REB,
+  // MQ_CONSUMER_STATUS__IDLE,
+  MQ_CONSUMER_STATUS__READY,
   MQ_CONSUMER_STATUS__LOST,
-  MQ_CONSUMER_STATUS__MODIFY
+  MQ_CONSUMER_STATUS__LOST_IN_REB,
+  MQ_CONSUMER_STATUS__LOST_REBD,
 };
 
-
 int32_t mndInitConsumer(SMnode *pMnode);
 void    mndCleanupConsumer(SMnode *pMnode);
 
@@ -43,6 +44,8 @@ SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup);
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
 SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
 
+int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndDef.h

@@ -88,6 +88,7 @@ typedef enum {
   TRN_TYPE_CREATE_STREAM = 1019,
   TRN_TYPE_DROP_STREAM = 1020,
   TRN_TYPE_ALTER_STREAM = 1021,
+  TRN_TYPE_CONSUMER_LOST = 1022,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -511,6 +512,7 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset)
   return buf;
 }
 
+#if 0
 typedef struct {
   char    key[TSDB_SUBSCRIBE_KEY_LEN];
   int32_t status;
@@ -641,6 +643,7 @@ static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
     pSub->unassignedVg = NULL;
   }
 }
+#endif
 
 typedef struct {
   char           name[TSDB_TOPIC_FNAME_LEN];
@@ -659,6 +662,7 @@ typedef struct {
   SSchemaWrapper schema;
 } SMqTopicObj;
 
+#if 0
 typedef struct {
   int64_t  consumerId;
   int64_t  connId;
@@ -713,7 +717,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
   buf = taosDecodeStringTo(buf, pConsumer->cgroup);
 
   buf = taosDecodeFixedI32(buf, &sz);
-  pConsumer->currentTopics = taosArrayInit(sz, sizeof(SMqConsumerObj));
+  pConsumer->currentTopics = taosArrayInit(sz, sizeof(void*));
   for (int32_t i = 0; i < sz; i++) {
     char* topic;
     buf = taosDecodeString(buf, &topic);
@@ -721,7 +725,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
   }
 
   buf = taosDecodeFixedI32(buf, &sz);
-  pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(SMqConsumerObj));
+  pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(void*));
   for (int32_t i = 0; i < sz; i++) {
     char* topic;
     buf = taosDecodeString(buf, &topic);
@@ -729,6 +733,132 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
   }
   return buf;
 }
+#endif
+
+enum {
+  CONSUMER_UPDATE__TOUCH = 1,
+  CONSUMER_UPDATE__ADD,
+  CONSUMER_UPDATE__REMOVE,
+  CONSUMER_UPDATE__LOST,
+  CONSUMER_UPDATE__MODIFY,
+};
+
+typedef struct {
+  int64_t consumerId;
+  char    cgroup[TSDB_CGROUP_LEN];
+  int8_t  updateType;  // used only for update
+  int32_t epoch;
+  int32_t status;
+  // hbStatus is not applicable to serialization
+  int32_t hbStatus;
+  // lock is used for topics update
+  SRWLatch lock;
+  SArray*  currentTopics;  // SArray<char*>
+#if 0
+  SArray*  waitingRebTopics;  // SArray<char*>
+#endif
+  SArray* rebNewTopics;      // SArray<char*>
+  SArray* rebRemovedTopics;  // SArray<char*>
+} SMqConsumerObj;
+
+SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);
+void            tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer);
+int32_t         tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer);
+void*           tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer);
+
+typedef struct {
+  int32_t vgId;
+  char*   qmsg;
+  // char    topic[TSDB_TOPIC_FNAME_LEN];
+  SEpSet epSet;
+} SMqVgEp;
+
+SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp);
+void     tDeleteSMqVgEp(SMqVgEp* pVgEp);
+int32_t  tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp);
+void*    tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp);
+
+typedef struct {
+  int64_t consumerId;  // -1 for unassigned
+  SArray* vgs;         // SArray<SMqVgEp*>
+} SMqConsumerEpInSub;
+
+SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub);
+void                tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub);
+int32_t             tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub);
+void*               tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub);
+
+typedef struct {
+  char      key[TSDB_SUBSCRIBE_KEY_LEN];
+  SRWLatch  lock;
+  int32_t   vgNum;
+  SHashObj* consumerHash;  // consumerId -> SMqConsumerEpInSub
+} SMqSubscribeObj;
+
+SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
+SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub);
+void             tDeleteSubscribeObj(SMqSubscribeObj* pSub);
+int32_t          tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub);
+void*            tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub);
+
+typedef struct {
+  int32_t epoch;
+  SArray* consumers;  // SArray<SMqConsumerEpInSub*>
+} SMqSubActionLogEntry;
+
+SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+void                  tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+int32_t               tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry);
+void*                 tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry);
+
+typedef struct {
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
+  SArray* logs;  // SArray<SMqSubActionLogEntry*>
+} SMqSubActionLogObj;
+
+SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+void                tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+int32_t             tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog);
+void*               tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog);
+
+typedef struct {
+  int64_t  consumerId;
+  char     cgroup[TSDB_CGROUP_LEN];
+  SRWLatch lock;
+  SArray*  vgs;  // SArray<SMqVgEp*>
+} SMqConsumerEpObj;
+
+SMqConsumerEpObj* tCloneSMqConsumerEpObj(const SMqConsumerEpObj* pConsumerEp);
+void              tDeleteSMqConsumerEpObj(SMqConsumerEpObj* pConsumerEp);
+int32_t           tEncodeSMqConsumerEpObj(void** buf, const SMqConsumerEpObj* pConsumerEp);
+void*             tDecodeSMqConsumerEpObj(const void* buf, SMqConsumerEpObj* pConsumerEp);
+
+typedef struct {
+  const SMqSubscribeObj* pOldSub;
+  const SMqTopicObj*     pTopic;
+  const SMqRebSubscribe* pRebInfo;
+} SMqRebInputObj;
+
+typedef struct {
+  int64_t  oldConsumerId;
+  int64_t  newConsumerId;
+  SMqVgEp* pVgEp;
+} SMqRebOutputVg;
+
+#if 0
+typedef struct {
+  int64_t consumerId;
+} SMqRebOutputConsumer;
+#endif
+
+typedef struct {
+  SArray*               rebVgs;            // SArray<SMqRebOutputVg>
+  SArray*               newConsumers;      // SArray<int64_t>
+  SArray*               removedConsumers;  // SArray<int64_t>
+  SArray*               touchedConsumers;  // SArray<int64_t>
+  SMqSubscribeObj*      pSub;
+  SMqSubActionLogEntry* pLogEntry;
+} SMqRebOutputObj;
 
 typedef struct {
   char     name[TSDB_TOPIC_FNAME_LEN];

source/dnode/mnode/impl/inc/mndOffset.h

@@ -31,7 +31,7 @@ void          mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset);
 SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset);
 SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw);
 
-int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
+int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
 
 static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
   return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);

source/dnode/mnode/impl/inc/mndSubscribe.h

@@ -29,6 +29,8 @@ SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const c
 SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key);
 void             mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
 
+int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndConsumer.c

@@ -19,8 +19,10 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndShow.h"
 #include "mndStb.h"
+#include "mndSubscribe.h"
 #include "mndTopic.h"
 #include "mndTrans.h"
 #include "mndUser.h"
@@ -31,6 +33,10 @@
 #define MND_CONSUMER_VER_NUMBER   1
 #define MND_CONSUMER_RESERVE_SIZE 64
 
+#define MND_CONSUMER_LOST_HB_CNT 3
+
+static int8_t mqInRebFlag = 0;
+
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
@@ -38,6 +44,11 @@ static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg);
 static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConsumer(SMnode *pMnode, void *pIter);
 
+static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
+static int32_t mndProcessAskEpReq(SNodeMsg *pMsg);
+static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
+static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg);
+
 int32_t mndInitConsumer(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_CONSUMER,
                      .keyType = SDB_KEY_INT64,
@@ -47,25 +58,397 @@ int32_t mndInitConsumer(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndConsumerActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
 
+  mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg);
   return sdbSetTable(pMnode->pSdb, table);
 }
 
 void mndCleanupConsumer(SMnode *pMnode) {}
 
-SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) {
+static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) {
-  SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
+  SMnode             *pMnode = pMsg->pNode;
-  if (pConsumer == NULL) {
+  SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont;
+  SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
+  ASSERT(pConsumer);
+
+  SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
+  pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
+
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_LOST, &pMsg->rpcMsg);
+  if (pTrans == NULL) goto FAIL;
+  if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+FAIL:
+  // TODO delete consumer
+  mndTransDrop(pTrans);
+  return -1;
+}
+
+static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
+  SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1);
+  if (pRebSub == NULL) {
+    pRebSub = tNewSMqRebSubscribe(key);
+    if (pRebSub == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
     }
+    taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe));
+  }
+  return pRebSub;
+}
 
-  pConsumer->recentRemovedTopics = taosArrayInit(1, sizeof(char *));
+static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
-  pConsumer->epoch = 1;
+  SMnode         *pMnode = pMsg->pNode;
-  pConsumer->consumerId = consumerId;
+  SSdb           *pSdb = pMnode->pSdb;
-  atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT);
+  SMqConsumerObj *pConsumer;
-  strcpy(pConsumer->cgroup, cgroup);
+  void           *pIter = NULL;
-  taosInitRWLatch(&pConsumer->lock);
+
-  return pConsumer;
+  // rebalance cannot be parallel
+  int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1);
+  if (old != 0) {
+    mInfo("mq rebalance already in progress, do nothing");
+    return 0;
+  }
+
+  SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
+  pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
+  // TODO set cleanfp
+  pRebMsg->mqInReb = &mqInRebFlag;
+
+  // iterate all consumers, find all modification
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
+    if (pIter == NULL) break;
+
+    int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
+    int32_t status = atomic_load_32(&pConsumer->status);
+    if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) {
+      SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg));
+
+      pLostMsg->consumerId = pConsumer->consumerId;
+      SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
+      pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_LOST;
+      pRpcMsg->pCont = pLostMsg;
+      pRpcMsg->contLen = sizeof(SMqConsumerLostMsg);
+      tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
+    }
+    if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) {
+      // do nothing
+    } else if (status == MQ_CONSUMER_STATUS__LOST) {
+      taosRLockLatch(&pConsumer->lock);
+      int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics);
+      for (int32_t i = 0; i < topicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
+      }
+      taosRUnLockLatch(&pConsumer->lock);
+    } else if (status == MQ_CONSUMER_STATUS__MODIFY) {
+      taosRLockLatch(&pConsumer->lock);
+      int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
+      for (int32_t i = 0; i < newTopicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId);
+      }
+
+      int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics);
+      for (int32_t i = 0; i < removedTopicNum; i++) {
+        char  key[TSDB_SUBSCRIBE_KEY_LEN];
+        char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i);
+        mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic);
+        SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
+        taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
+      }
+      taosRUnLockLatch(&pConsumer->lock);
+    } else {
+      // do nothing
+    }
+
+    mndReleaseConsumer(pMnode, pConsumer);
+  }
+
+  if (taosHashGetSize(pRebMsg->rebSubHash) != 0) {
+    mInfo("mq rebalance will be triggered");
+    SRpcMsg rpcMsg = {
+        .msgType = TDMT_MND_MQ_DO_REBALANCE,
+        .pCont = pRebMsg,
+        .contLen = sizeof(SMqDoRebalanceMsg),
+    };
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+  } else {
+    taosHashCleanup(pRebMsg->rebSubHash);
+    rpcFreeCont(pRebMsg);
+    mTrace("mq rebalance finished, no modification");
+    atomic_store_8(&mqInRebFlag, 0);
+  }
+  return 0;
+}
+
+static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
+  SMnode           *pMnode = pMsg->pNode;
+  SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
+  SMqCMGetSubEpRsp  rsp = {0};
+  int64_t           consumerId = be64toh(pReq->consumerId);
+  int32_t           epoch = ntohl(pReq->epoch);
+
+  SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId);
+  if (pConsumer == NULL) {
+    terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
+    return -1;
+  }
+
+  ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
+  /*int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);*/
+  atomic_store_32(&pConsumer->hbStatus, 0);
+
+  // 1. check consumer status
+  int32_t status = atomic_load_32(&pConsumer->status);
+
+  if (status == MQ_CONSUMER_STATUS__LOST) {
+    // TODO: recover consumer
+  }
+
+  if (status != MQ_CONSUMER_STATUS__READY) {
+    terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
+    return -1;
+  }
+
+  int32_t serverEpoch = atomic_load_32(&pConsumer->epoch);
+
+  // 2. check epoch, only send ep info when epoches do not match
+  if (epoch != serverEpoch) {
+    taosRLockLatch(&pConsumer->lock);
+    mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
+    int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics);
+
+    rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp));
+    if (rsp.topics == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      taosRUnLockLatch(&pConsumer->lock);
+      goto FAIL;
+    }
+
+    // handle all topic subscribed by the consumer
+    for (int32_t i = 0; i < numOfTopics; i++) {
+      char            *topic = taosArrayGetP(pConsumer->currentTopics, i);
+      SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
+
+      // txn guarantees pSub is created
+      ASSERT(pSub);
+      taosRLockLatch(&pSub->lock);
+
+      SMqSubTopicEp topicEp = {0};
+      strcpy(topicEp.topic, topic);
+
+      // 2.1 fetch topic schema
+      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+      ASSERT(pTopic);
+      taosRLockLatch(&pTopic->lock);
+      topicEp.schema.nCols = pTopic->schema.nCols;
+      topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema));
+      memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema));
+      taosRUnLockLatch(&pTopic->lock);
+      mndReleaseTopic(pMnode, pTopic);
+
+      // 2.2 iterate all vg assigned to the consumer of that topic
+      SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
+      int32_t             vgNum = taosArrayGetSize(pEpInSub->vgs);
+
+      topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
+      if (topicEp.vgs == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        taosRUnLockLatch(&pConsumer->lock);
+        goto FAIL;
+      }
+
+      for (int32_t j = 0; j < vgNum; j++) {
+        SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        char     offsetKey[TSDB_PARTITION_KEY_LEN];
+        mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
+        // 2.2.1 build vg ep
+        SMqSubVgEp vgEp = {
+            .epSet = pVgEp->epSet,
+            .vgId = pVgEp->vgId,
+            .offset = -1,
+        };
+
+        // 2.2.2 fetch vg offset
+        SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
+        if (pOffsetObj != NULL) {
+          vgEp.offset = atomic_load_64(&pOffsetObj->offset);
+          mndReleaseOffset(pMnode, pOffsetObj);
+        }
+        taosArrayPush(topicEp.vgs, &vgEp);
+      }
+      taosArrayPush(rsp.topics, &topicEp);
+
+      taosRUnLockLatch(&pSub->lock);
+      mndReleaseSubscribe(pMnode, pSub);
+    }
+    taosRUnLockLatch(&pConsumer->lock);
+  }
+  // encode rsp
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
+  void   *buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
+  ((SMqRspHead *)buf)->epoch = serverEpoch;
+  ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
+
+  void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+  tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
+
+  // release consumer and free memory
+  tDeleteSMqCMGetSubEpRsp(&rsp);
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  // send rsp
+  pMsg->pRsp = buf;
+  pMsg->rspLen = tlen;
+  return 0;
+FAIL:
+  tDeleteSMqCMGetSubEpRsp(&rsp);
+  mndReleaseConsumer(pMnode, pConsumer);
+  return -1;
+}
+
+int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer) {
+  SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
+  if (pCommitRaw == NULL) return -1;
+  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
+  if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
+
+static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
+  SMnode         *pMnode = pMsg->pNode;
+  char           *msgStr = pMsg->rpcMsg.pCont;
+  SCMSubscribeReq subscribe = {0};
+  tDeserializeSCMSubscribeReq(msgStr, &subscribe);
+  int64_t         consumerId = subscribe.consumerId;
+  char           *cgroup = subscribe.cgroup;
+  SMqConsumerObj *pConsumerOld = NULL;
+  SMqConsumerObj *pConsumerNew = NULL;
+
+  int32_t code = -1;
+  SArray *newSub = subscribe.topicNames;
+  taosArraySortString(newSub, taosArrayCompareString);
+
+  int32_t newTopicNum = taosArrayGetSize(newSub);
+  // check topic existance
+  for (int32_t i = 0; i < newTopicNum; i++) {
+    char        *topic = taosArrayGetP(newSub, i);
+    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+    if (pTopic == NULL) {
+      terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
+      goto SUBSCRIBE_OVER;
+    }
+    // TODO lock topic to prevent drop
+    mndReleaseTopic(pMnode, pTopic);
+  }
+
+  pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+  if (pConsumerOld == NULL) {
+    pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
+    /*pConsumerNew->waitingRebTopics = newSub;*/
+    pConsumerNew->rebNewTopics = newSub;
+    subscribe.topicNames = NULL;
+
+    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
+    if (pTrans == NULL) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+
+  } else {
+    /*taosRLockLatch(&pConsumerOld->lock);*/
+    int32_t status = atomic_load_32(&pConsumerOld->status);
+    if (status != MQ_CONSUMER_STATUS__READY) {
+      terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
+      goto SUBSCRIBE_OVER;
+    }
+
+    pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
+    if (pConsumerNew == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto SUBSCRIBE_OVER;
+    }
+    pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
+    /*pConsumerOld->waitingRebTopics = newSub;*/
+
+    int32_t oldTopicNum = 0;
+    if (pConsumerOld->currentTopics) {
+      oldTopicNum = taosArrayGetSize(pConsumerOld->currentTopics);
+    }
+
+    int32_t i = 0, j = 0;
+    while (i < oldTopicNum || j < newTopicNum) {
+      if (i >= oldTopicNum) {
+        char *newTopicCopy = strdup(taosArrayGetP(newSub, j));
+        taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
+        j++;
+        continue;
+      } else if (j >= newTopicNum) {
+        char *oldTopicCopy = strdup(taosArrayGetP(pConsumerOld->currentTopics, i));
+        taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
+        i++;
+        continue;
+      } else {
+        char *oldTopic = taosArrayGetP(pConsumerOld->currentTopics, i);
+        char *newTopic = taosArrayGetP(newSub, j);
+        int   comp = compareLenPrefixedStr(oldTopic, newTopic);
+        if (comp == 0) {
+          i++;
+          j++;
+          continue;
+        } else if (comp < 0) {
+          char *oldTopicCopy = strdup(oldTopic);
+          taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
+          i++;
+          continue;
+        } else {
+          char *newTopicCopy = strdup(newTopic);
+          taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
+          j++;
+          continue;
+        }
+      }
+    }
+
+    STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
+    if (pTrans == NULL) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+  }
+
+  code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+
+SUBSCRIBE_OVER:
+  if (pConsumerOld) {
+    /*taosRUnLockLatch(&pConsumerOld->lock);*/
+    mndReleaseConsumer(pMnode, pConsumerOld);
+  }
+  if (pConsumerNew) {
+    tDeleteSMqConsumerObj(pConsumerNew);
+  }
+  // TODO: replace with destroy subscribe msg
+  if (subscribe.topicNames) taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
+  return code;
 }
 
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
@@ -154,15 +537,141 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
   mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId);
+  tDeleteSMqConsumerObj(pConsumer);
   return 0;
 }
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
-  /*taosWLockLatch(&pOldConsumer->lock);*/
-  atomic_add_fetch_32(&pOldConsumer->epoch, 1);
-  /*taosWUnLockLatch(&pOldConsumer->lock);*/
 
+  taosWLockLatch(&pOldConsumer->lock);
+
+  if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) {
+    ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
+    SArray *tmp = pOldConsumer->rebNewTopics;
+    pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics;
+    pNewConsumer->rebNewTopics = tmp;
+
+    tmp = pOldConsumer->rebRemovedTopics;
+    pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics;
+    pNewConsumer->rebRemovedTopics = tmp;
+
+    pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
+    int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
+    pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i));
+      taosArrayPush(pNewConsumer->rebRemovedTopics, &topic);
+    }
+    pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) {
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
+    ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);
+    ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
+
+    char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
+    // not exist in current topic
+#if 1
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
+      ASSERT(strcmp(topic, addedTopic) != 0);
+    }
+#endif
+
+    // remove from new topic
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      if (strcmp(addedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->rebNewTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+
+    // add to current topic
+    taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
+    taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString);
+    // set status
+    if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
+      }
+    } else {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
+      }
+    }
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
+    ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);
+    char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
+
+    // not exist in new topic
+#if 1
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
+      ASSERT(strcmp(topic, removedTopic) != 0);
+    }
+#endif
+
+    // remove from removed topic
+    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) {
+      char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i);
+      if (strcmp(removedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->rebRemovedTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+
+    // remove from current topic
+    int32_t i = 0;
+    int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
+    for (i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
+      if (strcmp(removedTopic, topic) == 0) {
+        taosArrayRemove(pOldConsumer->currentTopics, i);
+        taosMemoryFree(topic);
+        break;
+      }
+    }
+    // must find the topic
+    ASSERT(i < sz);
+
+    // set status
+    if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
+      }
+    } else {
+      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
+      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
+        pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
+      }
+    }
+    atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+  }
+
+  taosWUnLockLatch(&pOldConsumer->lock);
   return 0;
 }
 

source/dnode/mnode/impl/src/mndDef.c

@@ -14,6 +14,403 @@
  */
 
 #include "mndDef.h"
+#include "mndConsumer.h"
+
+SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) {
+  SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
+  if (pConsumer == NULL) {
+    return NULL;
+  }
+
+  pConsumer->consumerId = consumerId;
+  memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN);
+
+  pConsumer->epoch = 0;
+  pConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
+  pConsumer->hbStatus = 0;
+
+  taosInitRWLatch(&pConsumer->lock);
+
+  pConsumer->currentTopics = taosArrayInit(0, sizeof(void *));
+#if 0
+  pConsumer->waitingRebTopics = NULL;
+#endif
+  pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
+  pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
+
+  if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) {
+    taosArrayDestroy(pConsumer->currentTopics);
+    taosArrayDestroy(pConsumer->rebNewTopics);
+    taosArrayDestroy(pConsumer->rebRemovedTopics);
+    taosMemoryFree(pConsumer);
+    return NULL;
+  }
+
+  return pConsumer;
+}
+
+void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
+  if (pConsumer->currentTopics) {
+    taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree);
+  }
+#if 0
+  if (pConsumer->waitingRebTopics) {
+    taosArrayDestroyP(pConsumer->waitingRebTopics, taosMemoryFree);
+  }
+#endif
+  if (pConsumer->rebNewTopics) {
+    taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree);
+  }
+  if (pConsumer->rebRemovedTopics) {
+    taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
+  }
+}
+
+int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
+  int32_t tlen = 0;
+  int32_t sz;
+  tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
+  tlen += taosEncodeString(buf, pConsumer->cgroup);
+  tlen += taosEncodeFixedI8(buf, pConsumer->updateType);
+  tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
+  tlen += taosEncodeFixedI32(buf, pConsumer->status);
+
+  // current topics
+  if (pConsumer->currentTopics) {
+    sz = taosArrayGetSize(pConsumer->currentTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->currentTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+#if 0
+  // waiting reb topics
+  if (pConsumer->waitingRebTopics) {
+    sz = taosArrayGetSize(pConsumer->waitingRebTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->waitingRebTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+#endif
+
+  // reb new topics
+  if (pConsumer->rebNewTopics) {
+    sz = taosArrayGetSize(pConsumer->rebNewTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->rebNewTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+  // reb removed topics
+  if (pConsumer->rebRemovedTopics) {
+    sz = taosArrayGetSize(pConsumer->rebRemovedTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->rebRemovedTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
+  return tlen;
+}
+
+void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
+  int32_t sz;
+  buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
+  buf = taosDecodeStringTo(buf, pConsumer->cgroup);
+  buf = taosDecodeFixedI8(buf, &pConsumer->updateType);
+  buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
+  buf = taosDecodeFixedI32(buf, &pConsumer->status);
+
+  // current topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->currentTopics, &topic);
+  }
+
+#if 0
+  // waiting reb topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->waitingRebTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->waitingRebTopics, &topic);
+  }
+#endif
+
+  // reb new topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->rebNewTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->rebNewTopics, &topic);
+  }
+
+  // reb removed topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->rebRemovedTopics, &topic);
+  }
+
+  return (void *)buf;
+}
+
+SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
+  SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
+  if (pVgEpNew == NULL) return NULL;
+  pVgEpNew->vgId = pVgEp->vgId;
+  pVgEpNew->qmsg = strdup(pVgEp->qmsg);
+  /*memcpy(pVgEpNew->topic, pVgEp->topic, TSDB_TOPIC_FNAME_LEN);*/
+  pVgEpNew->epSet = pVgEp->epSet;
+  return pVgEpNew;
+}
+
+void tDeleteSMqVgEp(SMqVgEp *pVgEp) { taosMemoryFree(pVgEp->qmsg); }
+
+int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
+  tlen += taosEncodeString(buf, pVgEp->qmsg);
+  /*tlen += taosEncodeString(buf, pVgEp->topic);*/
+  tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
+  return tlen;
+}
+
+void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
+  buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
+  buf = taosDecodeString(buf, &pVgEp->qmsg);
+  /*buf = taosDecodeStringTo(buf, pVgEp->topic);*/
+  buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
+  return (void *)buf;
+}
+
+SMqConsumerEpObj *tCloneSMqConsumerEpObj(const SMqConsumerEpObj *pConsumerEp) {
+  SMqConsumerEpObj *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEpObj));
+  if (pConsumerEpNew == NULL) return NULL;
+  pConsumerEpNew->consumerId = pConsumerEp->consumerId;
+  memcpy(pConsumerEpNew->cgroup, pConsumerEp->cgroup, TSDB_CGROUP_LEN);
+  taosInitRWLatch(&pConsumerEpNew->lock);
+  pConsumerEpNew->vgs = taosArrayDeepCopy(pConsumerEpNew->vgs, (FCopy)tCloneSMqVgEp);
+  return pConsumerEpNew;
+}
+
+void tDeleteSMqConsumerEpObj(SMqConsumerEpObj *pConsumerEp) {
+  taosArrayDestroyEx(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp);
+}
+
+int32_t tEncodeSMqConsumerEpObj(void **buf, const SMqConsumerEpObj *pConsumerEp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
+  tlen += taosEncodeString(buf, pConsumerEp->cgroup);
+  tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
+  return tlen;
+}
+
+void *tDecodeSMqConsumerEpObj(const void *buf, SMqConsumerEpObj *pConsumerEp) {
+  buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
+  buf = taosDecodeStringTo(buf, pConsumerEp->cgroup);
+  buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));
+  return (void *)buf;
+}
+
+SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) {
+  SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));
+  if (pEpInSubNew == NULL) return NULL;
+  pEpInSubNew->consumerId = pEpInSub->consumerId;
+  pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp);
+  return pEpInSubNew;
+}
+
+void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) {
+  taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp);
+}
+
+int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId);
+  int32_t sz = taosArrayGetSize(pEpInSub->vgs);
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i);
+    tlen += tEncodeSMqVgEp(buf, pVgEp);
+  }
+  /*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/
+  return tlen;
+}
+
+void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) {
+  buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId);
+  /*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  pEpInSub->vgs = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
+    buf = tDecodeSMqVgEp(buf, pVgEp);
+    taosArrayPush(pEpInSub->vgs, &pVgEp);
+  }
+
+  return (void *)buf;
+}
+
+SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
+  SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj));
+  if (pSubNew == NULL) return NULL;
+  memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN);
+  taosInitRWLatch(&pSubNew->lock);
+  pSubNew->vgNum = -1;
+  pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  // TODO set free fp
+  SMqConsumerEpInSub epInSub = {
+      .consumerId = -1,
+      .vgs = taosArrayInit(0, sizeof(void *)),
+  };
+  int64_t unexistKey = -1;
+  taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
+  return pSubNew;
+}
+
+SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
+  SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj));
+  if (pSubNew == NULL) return NULL;
+  memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
+  taosInitRWLatch(&pSubNew->lock);
+  pSubNew->vgNum = pSub->vgNum;
+  /*pSubNew->consumerEps = taosArrayDeepCopy(pSub->consumerEps, (FCopy)tCloneSMqConsumerEpInSub);*/
+  pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  /*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/
+  void               *pIter = NULL;
+  SMqConsumerEpInSub *pEpInSub = NULL;
+  while (1) {
+    pIter = taosHashIterate(pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    pEpInSub = (SMqConsumerEpInSub *)pIter;
+    SMqConsumerEpInSub newEp = {
+        .consumerId = pEpInSub->consumerId,
+        .vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp),
+    };
+    taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub));
+  }
+  return pSubNew;
+}
+
+void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
+  /*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/
+  taosHashCleanup(pSub->consumerHash);
+}
+
+int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pSub->key);
+  tlen += taosEncodeFixedI32(buf, pSub->vgNum);
+
+  void   *pIter = NULL;
+  int32_t sz = taosHashGetSize(pSub->consumerHash);
+  tlen += taosEncodeFixedI32(buf, sz);
+
+  int32_t cnt = 0;
+  while (1) {
+    pIter = taosHashIterate(pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub);
+    cnt++;
+  }
+  ASSERT(cnt == sz);
+  /*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/
+  return tlen;
+}
+
+void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
+  //
+  buf = taosDecodeStringTo(buf, pSub->key);
+  buf = taosDecodeFixedI32(buf, &pSub->vgNum);
+
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+
+  pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  for (int32_t i = 0; i < sz; i++) {
+    /*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/
+    SMqConsumerEpInSub epInSub = {0};
+    buf = tDecodeSMqConsumerEpInSub(buf, &epInSub);
+    taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
+  }
+
+  /*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/
+  return (void *)buf;
+}
+
+SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+  SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
+  if (pEntryNew == NULL) return NULL;
+  pEntryNew->epoch = pEntry->epoch;
+  pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub);
+  return pEntryNew;
+}
+
+void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub);
+}
+
+int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pEntry->epoch);
+  tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
+  return tlen;
+}
+void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
+  buf = taosDecodeFixedI32(buf, &pEntry->epoch);
+  buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+  return (void *)buf;
+}
+
+SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+  SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
+  if (pLogNew == NULL) return pLogNew;
+  memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
+  pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub);
+  return pLogNew;
+}
+
+void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub);
+}
+
+int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pLog->key);
+  tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry);
+  return tlen;
+}
+
+void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
+  buf = taosDecodeStringTo(buf, pLog->key);
+  buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+  return (void *)buf;
+}
 
 int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   int32_t sz = 0;

source/dnode/mnode/impl/src/mndOffset.c

@@ -130,8 +130,7 @@ OFFSET_DECODE_OVER:
   return pRow;
 }
 
-int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
+int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
-  int32_t code = 0;
   int32_t sz = taosArrayGetSize(vgs);
   for (int32_t i = 0; i < sz; i++) {
     SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i);
@@ -170,14 +169,23 @@ static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) {
     if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
       return -1;
     }
+    bool          create = false;
     SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
-    ASSERT(pOffsetObj);
+    if (pOffsetObj == NULL) {
+      pOffsetObj = taosMemoryMalloc(sizeof(SMqOffset));
+      memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN);
+      create = true;
+    }
     pOffsetObj->offset = pOffset->offset;
     SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
     sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
     mndTransAppendRedolog(pTrans, pOffsetRaw);
+    if (create) {
+      taosMemoryFree(pOffsetObj);
+    } else {
       mndReleaseOffset(pMnode, pOffsetObj);
     }
+  }
 
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -201,7 +209,7 @@ static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) {
 
 static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) {
   mTrace("offset:%s, perform update action", pOldOffset->key);
-  pOldOffset->offset = pNewOffset->offset;
+  atomic_store_64(&pOldOffset->offset, pNewOffset->offset);
   return 0;
 }
 

source/dnode/mnode/impl/src/mndScheduler.c

@@ -434,7 +434,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     return -1;
   }
 
-  ASSERT(pSub->vgNum == 0);
+  ASSERT(pSub->vgNum == -1);
+
+  pSub->vgNum = 0;
 
   int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
   if (levelNum != 1) {
@@ -453,6 +455,12 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
   }
   SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
 
+  int64_t             unexistKey = -1;
+  SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
+  ASSERT(pEpInSub);
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
   void* pIter = NULL;
   while (1) {
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@@ -466,24 +474,41 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     plan->execNode.nodeId = pVgroup->vgId;
     plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
+    SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
+    pVgEp->epSet = plan->execNode.epSet;
+    pVgEp->vgId = plan->execNode.nodeId;
+
+#if 0
     SMqConsumerEp consumerEp = {0};
     consumerEp.status = 0;
     consumerEp.consumerId = -1;
     consumerEp.epSet = plan->execNode.epSet;
     consumerEp.vgId = plan->execNode.nodeId;
+#endif
 
-    mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId);
+    mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId);
 
     int32_t msgLen;
-    if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
+    if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) {
       sdbRelease(pSdb, pVgroup);
       qDestroyQueryPlan(pPlan);
       terrno = TSDB_CODE_QRY_INVALID_INPUT;
       return -1;
     }
-    taosArrayPush(pSub->unassignedVg, &consumerEp);
+    taosArrayPush(pEpInSub->vgs, &pVgEp);
+
+    ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
+    /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
   }
 
+  ASSERT(pEpInSub->vgs->size > 0);
+  pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
+
+  ASSERT(pEpInSub->vgs->size > 0);
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
   qDestroyQueryPlan(pPlan);
 
   return 0;

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -40,30 +40,47 @@ enum {
   MQ_SUBSCRIBE_STATUS__DELETED,
 };
 
-static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
-
 static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
 static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
 static int32_t  mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub);
 
-static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
+/*static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);*/
-static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg);
+/*static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg);*/
 static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg);
 static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg);
-static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
+/*static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);*/
-static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg);
+/*static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg);*/
-static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg);
+/*static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg);*/
-static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);
+/*static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);*/
 
-static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
+static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg);
-                                      const SMqConsumerEp *pConsumerEp);
 
-static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
+static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
-                                      const char *topicName);
+  SSdbRaw *pRedoRaw = mndSubActionEncode(pSub);
-static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,
+  if (pRedoRaw == NULL) return -1;
-                                       const char *oldTopicName);
+  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
+
+static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
+  SSdbRaw *pCommitRaw = mndSubActionEncode(pSub);
+  if (pCommitRaw == NULL) return -1;
+  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
+  if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
+  return 0;
+}
+
+/*static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,*/
+/*const SMqConsumerEp *pConsumerEp);*/
+
+/*static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/
+/*const char *topicName);*/
+
+/*static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/
+/*const char *oldTopicName);*/
 
 int32_t mndInitSubscribe(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
@@ -74,16 +91,38 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndSubActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndSubActionDelete};
 
-  mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
+  /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);*/
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);
+  /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);*/
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);
+  /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);*/
-  mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);
+  /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);*/
-  mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
+  /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);*/
-  mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);
+  /*mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);*/
-  mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);
+  /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);*/
+  mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq);
   return sdbSetTable(pMnode->pSdb, table);
 }
 
+static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) {
+  SMqSubscribeObj *pSub = tNewSubscribeObj(subKey);
+  if (pSub == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
+  if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
+    tDeleteSubscribeObj(pSub);
+    taosMemoryFree(pSub);
+    return NULL;
+  }
+
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+
+  return pSub;
+}
+
+#if 0
 static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) {
   SMqSubscribeObj *pSub = tNewSubscribeObj();
   if (pSub == NULL) {
@@ -212,7 +251,63 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
 
   return 0;
 }
+#endif
 
+static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const char *subKey, const SMqRebOutputVg *pRebVg) {
+  SMqRebVgReq req = {0};
+  req.oldConsumerId = pRebVg->oldConsumerId;
+  req.newConsumerId = pRebVg->newConsumerId;
+  req.vgId = pRebVg->pVgEp->vgId;
+  req.qmsg = pRebVg->pVgEp->qmsg;
+  strncpy(req.subKey, subKey, TSDB_SUBSCRIBE_KEY_LEN);
+
+  int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req);
+  void   *buf = taosMemoryMalloc(tlen);
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  SMsgHead *pMsgHead = (SMsgHead *)buf;
+
+  pMsgHead->contLen = htonl(tlen);
+  pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId);
+
+  void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  tEncodeSMqRebVgReq(&abuf, &req);
+  *pBuf = buf;
+  *pLen = tlen;
+
+  return 0;
+}
+
+static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const char *subKey,
+                                        const SMqRebOutputVg *pRebVg) {
+  ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId);
+
+  void   *buf;
+  int32_t tlen;
+  if (mndBuildSubChangeReq(&buf, &tlen, subKey, pRebVg) < 0) {
+    return -1;
+  }
+
+  int32_t vgId = pRebVg->pVgEp->vgId;
+  SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
+
+  STransAction action = {0};
+  action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
+  action.pCont = buf;
+  action.contLen = tlen;
+  action.msgType = TDMT_VND_MQ_VG_CHANGE;
+
+  mndReleaseVgroup(pMnode, pVgObj);
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(buf);
+    return -1;
+  }
+  return 0;
+}
+
+#if 0
 static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
   SMnode           *pMnode = pMsg->pNode;
   SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
@@ -312,6 +407,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
   pMsg->rspLen = tlen;
   return 0;
 }
+#endif
 
 static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) {
   int32_t i = 0;
@@ -337,6 +433,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
   return pRebSub;
 }
 
+#if 0
 static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   SMnode            *pMnode = pMsg->pNode;
   SSdb              *pSdb = pMnode->pSdb;
@@ -408,7 +505,9 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   }
   return 0;
 }
+#endif
 
+#if 0
 static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
   SMnode            *pMnode = pMsg->pNode;
   SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont;
@@ -422,7 +521,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
     if (pIter == NULL) break;
     SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter;
     SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
-    taosMemoryFreeClear(pRebSub->key);
 
     mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum,
           (int32_t)taosArrayGetSize(pSub->unassignedVg));
@@ -562,6 +660,385 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
   mndTransDrop(pTrans);
   return 0;
 }
+#endif
+
+static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
+  if (pInput->pTopic != NULL) {
+    // create subscribe
+    pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key);
+    ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1);
+  } else {
+    pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub);
+  }
+  int32_t totalVgNum = pOutput->pSub->vgNum;
+
+  mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum);
+
+  // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
+  SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+
+  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
+  // 2. check and get actual removed consumers, put their vg into hash
+  int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+  int32_t actualRemoved = 0;
+  for (int32_t i = 0; i < removedNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    if (pEpInSub) {
+      ASSERT(consumerId == pEpInSub->consumerId);
+      actualRemoved++;
+      int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+      for (int32_t j = 0; j < consumerVgNum; j++) {
+        SMqVgEp       *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        SMqRebOutputVg outputVg = {
+            .oldConsumerId = consumerId,
+            .newConsumerId = -1,
+            .pVgEp = pVgEp,
+        };
+        taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+      }
+      taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+      // put into removed
+      taosArrayPush(pOutput->removedConsumers, &consumerId);
+    }
+  }
+  ASSERT(removedNum == actualRemoved);
+  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
+
+  // if previously no consumer, there are vgs not assigned
+  {
+    int64_t             unexistKey = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    for (int32_t i = 0; i < consumerVgNum; i++) {
+      SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+      SMqRebOutputVg rebOutput = {
+          .oldConsumerId = -1,
+          .newConsumerId = -1,
+          .pVgEp = pVgEp,
+      };
+      taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
+    }
+  }
+
+  // 3. calc vg number of each consumer
+  int32_t oldSz = 0;
+  if (pInput->pOldSub) {
+    oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1;
+  }
+  int32_t afterRebConsumerNum =
+      oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers);
+  int32_t minVgCnt = 0;
+  int32_t imbConsumerNum = 0;
+  // calc num
+  if (afterRebConsumerNum) {
+    minVgCnt = totalVgNum / afterRebConsumerNum;
+    imbConsumerNum = totalVgNum % afterRebConsumerNum;
+  }
+
+  // 4. first scan: remove consumer more than wanted, put to remove hash
+  int32_t imbCnt = 0;
+  void   *pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    if (pEpInSub->consumerId == -1) continue;
+    ASSERT(pEpInSub->consumerId > 0);
+    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    // all old consumers still existing are touched
+    // TODO optimize: touch only consumer whose vgs changed
+    taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId);
+    if (consumerVgNum > minVgCnt) {
+      if (imbCnt < imbConsumerNum) {
+        if (consumerVgNum == minVgCnt + 1) {
+          continue;
+        } else {
+          // pop until equal minVg + 1
+          while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) {
+            SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+            SMqRebOutputVg outputVg = {
+                .oldConsumerId = pEpInSub->consumerId,
+                .newConsumerId = -1,
+                .pVgEp = pVgEp,
+            };
+            taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+          }
+          imbCnt++;
+        }
+      } else {
+        // pop until equal minVg
+        while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) {
+          SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+          SMqRebOutputVg outputVg = {
+              .oldConsumerId = pEpInSub->consumerId,
+              .newConsumerId = -1,
+              .pVgEp = pVgEp,
+          };
+          taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+        }
+      }
+    }
+  }
+
+  // 5. add new consumer into sub
+  {
+    int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers);
+    for (int32_t i = 0; i < consumerNum; i++) {
+      int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
+      ASSERT(consumerId > 0);
+      SMqConsumerEpInSub newConsumerEp;
+      newConsumerEp.consumerId = consumerId;
+      newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
+      taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp,
+                  sizeof(SMqConsumerEpInSub));
+      /*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
+      /*ASSERT(pTestNew->consumerId == consumerId);*/
+      /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/
+      taosArrayPush(pOutput->newConsumers, &consumerId);
+    }
+  }
+
+  // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer.
+  // All related vg should be put into rebVgs
+  SMqRebOutputVg *pRebVg = NULL;
+  void           *pRemovedIter = NULL;
+  pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+    if (pIter == NULL) break;
+    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+    if (pEpInSub->consumerId == -1) continue;
+    ASSERT(pEpInSub->consumerId > 0);
+
+    // push until equal minVg
+    while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) {
+      // iter hash and find one vg
+      pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+      ASSERT(pRemovedIter);
+      pRebVg = (SMqRebOutputVg *)pRemovedIter;
+      // push
+      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pEpInSub->consumerId;
+      taosArrayPush(pOutput->rebVgs, pRebVg);
+    }
+
+#if 0
+    /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/
+    if (imbCnt < imbConsumerNum) {
+      imbCnt++;
+      // push until equal minVg + 1
+      while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt + 1) {
+        // iter hash and find one vg
+        pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+        ASSERT(pRemovedIter);
+        pRebVg = (SMqRebOutputVg *)pRemovedIter;
+        // push
+        taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+        pRebVg->newConsumerId = pEpInSub->consumerId;
+        taosArrayPush(pOutput->rebVgs, pRebVg);
+      }
+    } else {
+      // push until equal minVg
+      while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) {
+        // iter hash and find one vg
+        pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+        ASSERT(pRemovedIter);
+        pRebVg = (SMqRebOutputVg *)pRemovedIter;
+        // push
+        taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+        pRebVg->newConsumerId = pEpInSub->consumerId;
+        taosArrayPush(pOutput->rebVgs, pRebVg);
+      }
+    }
+#endif
+  }
+
+  // 7. handle unassigned vg
+  if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) {
+    // if has consumer, assign all left vg
+    while (1) {
+      pRemovedIter = taosHashIterate(pHash, pRemovedIter);
+      if (pRemovedIter == NULL) break;
+      pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
+      ASSERT(pIter);
+      pRebVg = (SMqRebOutputVg *)pRemovedIter;
+      SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
+      if (pEpInSub->consumerId == -1) continue;
+      ASSERT(pEpInSub->consumerId > 0);
+      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pEpInSub->consumerId;
+      taosArrayPush(pOutput->rebVgs, pRebVg);
+    }
+  } else {
+    // if all consumer is removed, put all vg into unassigned
+    int64_t             unexistKey = -1;
+    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
+    ASSERT(pEpInSub);
+    ASSERT(pEpInSub->consumerId == -1);
+
+    pIter = NULL;
+    SMqRebOutputVg *pRebOutput = NULL;
+    while (1) {
+      pIter = taosHashIterate(pHash, pIter);
+      if (pIter == NULL) break;
+      pRebOutput = (SMqRebOutputVg *)pIter;
+      ASSERT(pRebOutput->newConsumerId == -1);
+      taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp);
+      taosArrayPush(pOutput->rebVgs, pRebOutput);
+    }
+  }
+
+  // 8. generate logs
+
+  // 9. clear
+  taosHashCleanup(pHash);
+
+  return 0;
+}
+
+static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebOutputObj *pOutput) {
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg);
+  if (pTrans == NULL) {
+    return -1;
+  }
+  // make txn:
+  // 1. redo action: action to all vg
+  const SArray *rebVgs = pOutput->rebVgs;
+  int32_t       vgNum = taosArrayGetSize(rebVgs);
+  for (int32_t i = 0; i < vgNum; i++) {
+    SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i);
+    if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub->key, pRebVg) < 0) {
+      goto REB_FAIL;
+    }
+  }
+
+  // 2. redo log: subscribe and vg assignment
+  // subscribe
+  if (mndSetSubRedoLogs(pMnode, pTrans, pOutput->pSub) != 0) {
+    goto REB_FAIL;
+  }
+
+  // 3. commit log: consumer to update status and epoch
+  // 3.1 set touched consumer
+  int32_t consumerNum = taosArrayGetSize(pOutput->touchedConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t         consumerId = *(int64_t *)taosArrayGet(pOutput->touchedConsumers, i);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH;
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+  // 3.2 set new consumer
+  consumerNum = taosArrayGetSize(pOutput->newConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__ADD;
+    char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
+    char  cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    taosArrayPush(pConsumerNew->rebNewTopics, &topic);
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+
+  // 3.3 set removed consumer
+  consumerNum = taosArrayGetSize(pOutput->removedConsumers);
+  for (int32_t i = 0; i < consumerNum; i++) {
+    int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i);
+    ASSERT(consumerId > 0);
+    SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+    SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
+    pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE;
+    char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
+    char  cgroup[TSDB_CGROUP_LEN];
+    mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup);
+    taosArrayPush(pConsumerNew->rebRemovedTopics, &topic);
+    mndReleaseConsumer(pMnode, pConsumerOld);
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+      goto REB_FAIL;
+    }
+  }
+  // 4. commit log: modification log
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+
+REB_FAIL:
+  mndTransDrop(pTrans);
+  return -1;
+}
+
+static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
+  SMnode            *pMnode = pMsg->pNode;
+  SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont;
+  void              *pIter = NULL;
+
+  mInfo("mq rebalance start");
+
+  while (1) {
+    pIter = taosHashIterate(pReq->rebSubHash, pIter);
+    if (pIter == NULL) break;
+    SMqRebInputObj rebInput = {0};
+
+    SMqRebOutputObj rebOutput = {0};
+    rebOutput.newConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.removedConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.touchedConsumers = taosArrayInit(0, sizeof(void *));
+    rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg));
+
+    SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter;
+    SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
+
+    if (pSub == NULL) {
+      // split sub key and extract topic
+      char topic[TSDB_TOPIC_FNAME_LEN];
+      char cgroup[TSDB_CGROUP_LEN];
+      mndSplitSubscribeKey(pRebSub->key, topic, cgroup);
+      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
+      ASSERT(pTopic);
+      taosRLockLatch(&pTopic->lock);
+      rebInput.pTopic = pTopic;
+    }
+
+    rebInput.pRebInfo = pRebSub;
+    rebInput.pOldSub = pSub;
+
+    // TODO replace assert with error check
+    ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
+    // if add more consumer to balanced subscribe,
+    // possibly no vg is changed
+    /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
+    ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
+
+    if (rebInput.pTopic) {
+      SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic;
+      taosRUnLockLatch(&pTopic->lock);
+      mndReleaseTopic(pMnode, pTopic);
+    } else {
+      mndReleaseSubscribe(pMnode, pSub);
+    }
+  }
+
+  // reset flag
+  atomic_store_8(pReq->mqInReb, 0);
+  mInfo("mq rebalance completed successfully");
+  taosHashCleanup(pReq->rebSubHash);
+
+  return 0;
+}
 
 static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp) {
@@ -697,16 +1174,23 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) {
 
 static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) {
   mTrace("subscribe:%s, perform delete action", pSub->key);
-  tDeleteSMqSubscribeObj(pSub);
+  tDeleteSubscribeObj(pSub);
   return 0;
 }
 
 static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) {
   mTrace("subscribe:%s, perform update action", pOldSub->key);
+  taosWLockLatch(&pOldSub->lock);
+
+  SHashObj *tmp = pOldSub->consumerHash;
+  pOldSub->consumerHash = pNewSub->consumerHash;
+  pNewSub->consumerHash = tmp;
+
+  taosWUnLockLatch(&pOldSub->lock);
   return 0;
 }
 
-static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
+int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
   int32_t tlen = strlen(cgroup);
   memcpy(key, cgroup, tlen);
   key[tlen] = TMQ_SEPARATOR;
@@ -739,6 +1223,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) {
   sdbRelease(pSdb, pSub);
 }
 
+#if 0
 static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   SMnode         *pMnode = pMsg->pNode;
   char           *msgStr = pMsg->rpcMsg.pCont;
@@ -901,6 +1386,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer);
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
+#endif
 
 static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) {
   mndTransProcessRsp(pRsp);

source/dnode/mnode/impl/src/mnode.c

@@ -22,12 +22,14 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndFunc.h"
+#include "mndGrant.h"
 #include "mndInfoSchema.h"
-#include "mndPerfSchema.h"
 #include "mndMnode.h"
 #include "mndOffset.h"
+#include "mndPerfSchema.h"
 #include "mndProfile.h"
 #include "mndQnode.h"
+#include "mndQuery.h"
 #include "mndShow.h"
 #include "mndSma.h"
 #include "mndSnode.h"
@@ -40,8 +42,6 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
-#include "mndQuery.h"
-#include "mndGrant.h"
 
 #define MQ_TIMER_MS    3000
 #define TRNAS_TIMER_MS 6000

source/dnode/vnode/src/inc/tq.h

@@ -16,6 +16,13 @@
 #ifndef _TD_VNODE_TQ_H_
 #define _TD_VNODE_TQ_H_
 
+#include "executor.h"
+#include "os.h"
+#include "thash.h"
+#include "tmsg.h"
+#include "ttimer.h"
+#include "wal.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -30,12 +37,6 @@ extern "C" {
 #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
-enum {
-  TQ_STREAM_TOKEN__DATA = 1,
-  TQ_STREAM_TOKEN__WATERMARK,
-  TQ_STREAM_TOKEN__CHECKPOINT,
-};
-
 #define TQ_BUFFER_SIZE 4
 
 #define TQ_BUCKET_MASK 0xFF
@@ -151,18 +152,23 @@ typedef struct {
 } STqMetaStore;
 
 typedef struct {
-  SMemAllocatorFactory* pAllocatorFactory;
+  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
-  SMemAllocator*        pAllocator;
+  int64_t consumerId;
-} STqMemRef;
+  int32_t epoch;
+  char*   qmsg;
+  // SRWLatch        lock;
+  SWalReadHandle* pReadHandle;
+  // number should be identical to fetch thread num
+  qTaskInfo_t task[4];
+} STqExec;
 
 struct STQ {
   // the collection of groups
   // the handle of meta kvstore
   bool          writeTrigger;
   char*         path;
-  STqMemRef     tqMemRef;
   STqMetaStore* tqMeta;
-  // STqPushMgr*   tqPushMgr;
+  SHashObj*     tqMetaNew;  // subKey -> tqExec
   SHashObj*     pStreamTasks;
   SVnode*       pVnode;
   SWal*         pWal;
@@ -230,10 +236,6 @@ typedef struct {
   // TODO sync function
 } STqStreamPusher;
 
-typedef struct {
-  int8_t type;  // mq or stream
-} STqPusher;
-
 typedef struct {
   SHashObj* pHash;  // <id, STqPush*>
 } STqPushMgr;
@@ -257,9 +259,10 @@ int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
 int tqCommit(STQ*);
 
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
-int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessRebReq(STQ* pTq, char* msg);
+// int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
-int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
+// int32_t tqProcessRebReq(STQ* pTq, char* msg);
+// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);

source/dnode/vnode/src/tq/tq.c

@@ -33,20 +33,10 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta) {
                             (FTqDelete)taosMemoryFree, 0);
   if (pTq->tqMeta == NULL) {
     taosMemoryFree(pTq);
-#if 0
-    allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator);
-#endif
     return NULL;
   }
 
-#if 0
+  pTq->tqMetaNew = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
-  pTq->tqPushMgr = tqPushMgrOpen();
-  if (pTq->tqPushMgr == NULL) {
-    // free store
-    taosMemoryFree(pTq);
-    return NULL;
-  }
-#endif
 
   pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
 
@@ -241,16 +231,15 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 
   return 0;
 }
-#if 0
+
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
-  SMqPollReq* pReq = pMsg->pCont;
+  SMqPollReqV2* pReq = pMsg->pCont;
   int64_t       consumerId = pReq->consumerId;
-  int64_t     fetchOffset;
-  int64_t     blockingTime = pReq->blockingTime;
   int32_t       reqEpoch = pReq->epoch;
+  int64_t       fetchOffset;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-    fetchOffset = 0;
+    fetchOffset = walGetFirstVer(pTq->pWal);
   } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
     fetchOffset = walGetLastVer(pTq->pWal);
   } else {
@@ -260,65 +249,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
          TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
-  SMqPollRsp rsp = {
+  STqExec* pExec = taosHashGet(pTq->tqMetaNew, pReq->subKey, strlen(pReq->subKey));
-      /*.consumerId = consumerId,*/
+  ASSERT(pExec);
-      .numOfTopics = 0,
-      .pBlockData = NULL,
-  };
 
-  STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
+  int32_t consumerEpoch = atomic_load_32(&pExec->epoch);
-  if (pConsumer == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch);
   while (consumerEpoch < reqEpoch) {
-    consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch);
+    consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch);
   }
 
-  STqTopic* pTopic = NULL;
+  SMqDataBlkRsp rsp = {0};
-  int32_t   sz = taosArrayGetSize(pConsumer->topics);
-  for (int32_t i = 0; i < sz; i++) {
-    STqTopic* topic = taosArrayGet(pConsumer->topics, i);
-    // TODO race condition
-    ASSERT(pConsumer->consumerId == consumerId);
-    if (strcmp(topic->topicName, pReq->topic) == 0) {
-      pTopic = topic;
-      break;
-    }
-  }
-  if (pTopic == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
-          TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
-         TD_VID(pTq->pVnode));
-
   rsp.reqOffset = pReq->currentOffset;
-  rsp.skipLogNum = 0;
+  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
+  rsp.blockData = taosArrayInit(0, sizeof(void*));
 
   while (1) {
-    /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
+    consumerEpoch = atomic_load_32(&pExec->epoch);
-    // TODO
-    consumerEpoch = atomic_load_32(&pConsumer->epoch);
     if (consumerEpoch > reqEpoch) {
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
              consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
+
     SWalReadHead* pHead;
-    if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
+    if (walReadWithHandle_s(pExec->pReadHandle, fetchOffset, &pHead) < 0) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
@@ -326,101 +279,88 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
              TD_VID(pTq->pVnode), fetchOffset);
       break;
     }
+
     vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
            TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
-    /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
+
-    /*pHead = pTopic->pReadhandle->pHead;*/
     if (pHead->msgType == TDMT_VND_SUBMIT) {
       SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
-      qTaskInfo_t task = pTopic->buffer.output[workerId].task;
+      qTaskInfo_t task = pExec->task[workerId];
       ASSERT(task);
       qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
-      SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
       while (1) {
         SSDataBlock* pDataBlock = NULL;
-        uint64_t     ts;
+        uint64_t     ts = 0;
         if (qExecTask(task, &pDataBlock, &ts) < 0) {
-          ASSERT(false);
+          ASSERT(0);
+        }
+        if (pDataBlock == NULL) break;
+
+        ASSERT(pDataBlock->info.rows != 0);
+        ASSERT(pDataBlock->info.numOfCols != 0);
+
+        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
+        void*              buf = taosMemoryCalloc(1, dataStrLen);
+        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+        pRetrieve->useconds = ts;
+        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+        pRetrieve->compressed = 0;
+        pRetrieve->completed = 1;
+        pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
+
+        // TODO enable compress
+        int32_t actualLen = 0;
+        blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
+        actualLen += sizeof(SRetrieveTableRsp);
+        ASSERT(actualLen <= dataStrLen);
+        taosArrayPush(rsp.blockDataLen, &actualLen);
+        taosArrayPush(rsp.blockData, &buf);
+        rsp.blockNum++;
       }
-        if (pDataBlock == NULL) {
-          /*pos = fetchOffset % TQ_BUFFER_SIZE;*/
-          break;
     }
 
-        taosArrayPush(pRes, pDataBlock);
+    // TODO batch optimization
-      }
+    if (rsp.blockNum != 0) break;
-
-      if (taosArrayGetSize(pRes) == 0) {
-        vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
-               pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
-        fetchOffset++;
     rsp.skipLogNum++;
-        taosArrayDestroy(pRes);
+    fetchOffset++;
-        continue;
   }
-      rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper;
+
+  ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
+  ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
+
+  if (rsp.blockNum != 0)
     rsp.rspOffset = fetchOffset;
+  else
+    rsp.rspOffset = fetchOffset - 1;
 
-      rsp.numOfTopics = 1;
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
-      rsp.pBlockData = pRes;
-
-      int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     pMsg->code = -1;
-        taosMemoryFree(pHead);
     return -1;
   }
+
   ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
   ((SMqRspHead*)buf)->epoch = pReq->epoch;
   ((SMqRspHead*)buf)->consumerId = consumerId;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-      tEncodeSMqPollRsp(&abuf, &rsp);
+  tEncodeSMqDataBlkRsp(&abuf, &rsp);
-      /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/
-      pMsg->pCont = buf;
-      pMsg->contLen = tlen;
-      pMsg->code = 0;
-      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
-             pHead->msgType, consumerId, pReq->epoch);
-      tmsgSendRsp(pMsg);
-      taosMemoryFree(pHead);
-      return 0;
-    } else {
-      taosMemoryFree(pHead);
-      fetchOffset++;
-      rsp.skipLogNum++;
-    }
-  }
-
-  /*if (blockingTime != 0) {*/
-  /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
-  /*} else {*/
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
-  void*   buf = rpcMallocCont(tlen);
-  if (buf == NULL) {
-    pMsg->code = -1;
-    return -1;
-  }
-  ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
-  ((SMqRspHead*)buf)->epoch = pReq->epoch;
-  rsp.rspOffset = fetchOffset - 1;
-
-  void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqPollRsp(&abuf, &rsp);
-  rsp.pBlockData = NULL;
   pMsg->pCont = buf;
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
-         pReq->epoch);
-  /*}*/
 
+  vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+         TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
+
+  // TODO destroy
+  taosArrayDestroy(rsp.blockData);
+  taosArrayDestroy(rsp.blockDataLen);
   return 0;
 }
-#endif
 
+#if 0
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
@@ -429,7 +369,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   int32_t     reqEpoch = pReq->epoch;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-    fetchOffset = 0;
+    fetchOffset = walGetFirstVer(pTq->pWal);
   } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
     fetchOffset = walGetLastVer(pTq->pWal);
   } else {
@@ -628,7 +568,57 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   return 0;
 }
+#endif
 
+// TODO: persist meta into tdb
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
+  SMqRebVgReq req;
+  tDecodeSMqRebVgReq(msg, &req);
+  // todo lock
+  STqExec* pExec = taosHashGet(pTq->tqMetaNew, req.subKey, strlen(req.subKey));
+  if (pExec == NULL) {
+    ASSERT(req.oldConsumerId == -1);
+    ASSERT(req.newConsumerId != -1);
+    STqExec exec = {0};
+    pExec = &exec;
+    /*taosInitRWLatch(&pExec->lock);*/
+
+    memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
+    pExec->consumerId = req.newConsumerId;
+    pExec->epoch = -1;
+    pExec->qmsg = req.qmsg;
+    req.qmsg = NULL;
+    pExec->pReadHandle = walOpenReadHandle(pTq->pVnode->pWal);
+    for (int32_t i = 0; i < 4; i++) {
+      STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
+      SReadHandle    handle = {
+             .reader = pReadHandle,
+             .meta = pTq->pVnodeMeta,
+      };
+      pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);
+      ASSERT(pExec->task[i]);
+    }
+    taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
+    return 0;
+  } else {
+    /*if (req.newConsumerId != -1) {*/
+    /*taosWLockLatch(&pExec->lock);*/
+    ASSERT(pExec->consumerId == req.oldConsumerId);
+    // TODO handle qmsg and exec modification
+    atomic_store_32(&pExec->epoch, -1);
+    atomic_store_64(&pExec->consumerId, req.newConsumerId);
+    atomic_add_fetch_32(&pExec->epoch, 1);
+    /*taosWUnLockLatch(&pExec->lock);*/
+    return 0;
+    /*} else {*/
+    // TODO
+    /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/
+    /*return 0;*/
+    /*}*/
+  }
+}
+
+#if 0
 int32_t tqProcessRebReq(STQ* pTq, char* msg) {
   SMqMVRebReq req = {0};
   terrno = TSDB_CODE_SUCCESS;
@@ -747,6 +737,7 @@ int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) {
   terrno = TSDB_CODE_SUCCESS;
   return 0;
 }
+#endif
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
   if (pTask->execType == TASK_EXEC__NONE) return 0;

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #include "vnodeInt.h"
-// TODO:replace by an abstract file layer
 // #include <fcntl.h>
 // #include <string.h>
 // #include <unistd.h>

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -90,6 +90,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       vnodeProcessSubmitReq(pVnode, ptr, pRsp);
       break;
     /* TQ */
+    case TDMT_VND_MQ_VG_CHANGE:
+      if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
+        // TODO: handle error
+      }
+      break;
+#if 0
     case TDMT_VND_MQ_SET_CONN: {
       if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
         // TODO: handle error
@@ -103,6 +110,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
       }
     } break;
+#endif
     case TDMT_VND_TASK_DEPLOY: {
       if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {

source/libs/function/inc/fnLog.h

@@ -0,0 +1,24 @@
+//
+// Created by slzhou on 22-4-20.
+//
+
+#ifndef TDENGINE_FNLOG_H
+#define TDENGINE_FNLOG_H
+#include "tlog.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define fnFatal(...) { if (fnDebugFlag & DEBUG_FATAL) { taosPrintLog("FN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
+#define fnError(...) { if (fnDebugFlag & DEBUG_ERROR) { taosPrintLog("FN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
+#define fnWarn(...)  { if (fnDebugFlag & DEBUG_WARN)  { taosPrintLog("FN WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
+#define fnInfo(...)  { if (fnDebugFlag & DEBUG_INFO)  { taosPrintLog("FN ", DEBUG_INFO, 255, __VA_ARGS__); }}
+#define fnDebug(...) { if (fnDebugFlag & DEBUG_DEBUG) { taosPrintLog("FN ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
+#define fnTrace(...) { if (fnDebugFlag & DEBUG_TRACE) { taosPrintLog("FN ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_FNLOG_H

source/libs/function/inc/tudf.h

@@ -26,6 +26,9 @@
 extern "C" {
 #endif
 
+#define UDF_LISTEN_PIPE_NAME_LEN 32
+#define UDF_LISTEN_PIPE_NAME_PREFIX "udf.sock."
+
 //======================================================================================
 //begin API to taosd and qworker
 
@@ -35,17 +38,28 @@ enum {
   UDFC_CODE_PIPE_READ_ERR = -3,
 };
 
+/*TODO: no api for dnode startudfd/stopudfd*/
 /**
- * start udf dameon service
+ * start udfd dameon service
- * @return error code
  */
-int32_t startUdfService();
+int32_t startUdfd(int32_t dnodeId);
 
 /**
- * stop udf dameon service
+ * stop udfd dameon service
+ */
+int32_t stopUdfd(int32_t dnodeId);
+
+/**
+ * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
  * @return error code
  */
-int32_t stopUdfService();
+int32_t createUdfdProxy(int32_t dnodeId);
+
+/**
+ * destroy udfd proxy
+ * @return error code
+ */
+int32_t destroyUdfdProxy(int32_t dnodeId);
 
 typedef void *UdfHandle;
 
@@ -101,7 +115,6 @@ typedef struct SUdfInterBuf {
   char* buf;
 } SUdfInterBuf;
 
-//TODO: translate these calls to callUdf
 // output: interBuf
 int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf);
 // input: block, state

source/libs/function/inc/udfc.h

@@ -32,9 +32,9 @@ typedef struct SUdfInfo {
 
 typedef void *UdfHandle;
 
-int32_t startUdfService();
+int32_t createUdfdProxy();
 
-int32_t stopUdfService();
+int32_t destroyUdfdProxy();
 
 //int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles);
 

source/libs/function/src/tudf.c

@@ -200,10 +200,10 @@ int64_t gUdfTaskSeqNum = 0;
 
 enum {
   UDFC_STATE_INITAL = 0, // initial state
-  UDFC_STATE_STARTNG, // starting after startUdfService
+  UDFC_STATE_STARTNG, // starting after createUdfdProxy
   UDFC_STATE_READY, // started and begin to receive quests
   UDFC_STATE_RESTARTING, // udfd abnormal exit. cleaning up and restart.
-  UDFC_STATE_STOPPING, // stopping after stopUdfService
+  UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy
   UDFC_STATUS_FINAL, // stopped
 };
 int8_t gUdfcState = UDFC_STATE_INITAL;
@@ -929,7 +929,7 @@ void udfStopAsyncCb(uv_async_t *async) {
   }
 }
 
-int32_t startUdfd();
+int32_t udfcSpawnUdfd();
 
 void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) {
   //TODO: pipe close will be first received
@@ -944,12 +944,12 @@ void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) {
   if (gUdfcState == UDFC_STATE_READY) {
     gUdfcState = UDFC_STATE_RESTARTING;
     //TODO: asynchronous without blocking. how to do it
-    cleanUpUvTasks();
+    //cleanUpUvTasks();
-    startUdfd();
+    udfcSpawnUdfd();
   }
 }
 
-int32_t startUdfd() {
+int32_t udfcSpawnUdfd() {
   //TODO: path
   uv_process_options_t options = {0};
   static char path[256] = {0};
@@ -979,9 +979,6 @@ int32_t startUdfd() {
 void constructUdfService(void *argsThread) {
   uv_loop_init(&gUdfdLoop);
 
-  //TODO spawn error
-  startUdfd();
-
   uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb);
   uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb);
   uv_mutex_init(&gUdfTaskQueueMutex);
@@ -994,7 +991,7 @@ void constructUdfService(void *argsThread) {
 }
 
 
-int32_t startUdfService() {
+int32_t createUdfdProxy(int32_t dnodeId) {
   gUdfcState = UDFC_STATE_STARTNG;
   uv_barrier_init(&gUdfInitBarrier, 2);
   uv_thread_create(&gUdfLoopThread, constructUdfService, 0);
@@ -1002,12 +999,12 @@ int32_t startUdfService() {
   return 0;
 }
 
-int32_t stopUdfService() {
+int32_t destroyUdfdProxy(int32_t dnodeId) {
   gUdfcState = UDFC_STATE_STOPPING;
   uv_barrier_destroy(&gUdfInitBarrier);
-  if (gUdfcState == UDFC_STATE_STOPPING) {
+//  if (gUdfcState == UDFC_STATE_STOPPING) {
-    uv_process_kill(&gUdfdProcess, SIGINT);
+//    uv_process_kill(&gUdfdProcess, SIGINT);
-  }
+//  }
   uv_async_send(&gUdfLoopStopAsync);
   uv_thread_join(&gUdfLoopThread);
   uv_mutex_destroy(&gUdfTaskQueueMutex);

source/libs/function/src/udfd.c

@@ -12,10 +12,10 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
 #include "uv.h"
 #include "os.h"
-#include "tlog.h"
+#include "fnLog.h"
+#include "thash.h"
 
 #include "tudf.h"
 #include "tudfInt.h"
@@ -25,7 +25,18 @@
 #include "tmsg.h"
 #include "trpc.h"
 
-static uv_loop_t *loop;
+typedef struct SUdfdContext {
+  uv_loop_t *loop;
+  char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN];
+  void      *clientRpc;
+
+  uv_mutex_t udfsMutex;
+  SHashObj* udfsHash;
+
+  bool printVersion;
+} SUdfdContext;
+
+SUdfdContext global;
 
 typedef struct SUdfdUvConn {
   uv_stream_t *client;
@@ -41,11 +52,22 @@ typedef struct SUvUdfWork {
   uv_buf_t     output;
 } SUvUdfWork;
 
+typedef enum {
+  UDF_STATE_INIT = 0,
+  UDF_STATE_LOADING,
+  UDF_STATE_READY,
+  UDF_STATE_UNLOADING
+} EUdfState;
+
 typedef struct SUdf {
   int32_t refCount;
+  EUdfState state;
+  uv_mutex_t lock;
+  uv_cond_t condReady;
 
   char   name[16];
   int8_t type;
+  char path[PATH_MAX];
 
   uv_lib_t              lib;
   TUdfScalarProcFunc    scalarProcFunc;
@@ -58,6 +80,25 @@ typedef struct SUdfHandle {
   SUdf *udf;
 } SUdfHandle;
 
+int32_t udfdLoadUdf(char* udfName, SUdf* udf) {
+    strcpy(udf->name, udfName);
+
+    int   err = uv_dlopen(udf->path, &udf->lib);
+    if (err != 0) {
+      fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
+      // TODO set error
+    }
+    //TODO: find all the functions
+    char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
+    strcpy(normalFuncName, udfName);
+    uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc));
+    char  freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
+    char *freeSuffix = "_free";
+    strncpy(freeFuncName, normalFuncName, strlen(normalFuncName));
+    strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
+    uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
+    return 0;
+}
 
 void udfdProcessRequest(uv_work_t *req) {
   SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
@@ -66,33 +107,44 @@ void udfdProcessRequest(uv_work_t *req) {
 
   switch (request.type) {
     case UDF_TASK_SETUP: {
-            debugPrint("%s", "process setup request");
+      //TODO: tracable id from client. connect, setup, call, teardown
-            SUdf *udf = taosMemoryMalloc(sizeof(SUdf));
+      fnInfo("%"PRId64" setup request. udf name: %s", request.seqNum, request.setup.udfName);
-            udf->refCount = 0;
       SUdfSetupRequest *setup = &request.setup;
-            strcpy(udf->name, setup->udfName);
+
-            //TODO: retrive udf info from mnode
+      SUdf* udf = NULL;
-            char* path = "libudf1.so";
+      uv_mutex_lock(&global.udfsMutex);
-            int err = uv_dlopen(path, &udf->lib);
+      SUdf** udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN);
-            if (err != 0) {
+      if (*udfInHash) {
-                debugPrint("can not load library %s. error: %s", path, uv_strerror(err));
+        ++(*udfInHash)->refCount;
-                //TODO set error
+        udf = *udfInHash;
+        uv_mutex_unlock(&global.udfsMutex);
+      } else {
+        SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf));
+        udfNew->refCount = 1;
+        udfNew->state = UDF_STATE_INIT;
+
+        uv_mutex_init(&udfNew->lock);
+        uv_cond_init(&udfNew->condReady);
+        udf = udfNew;
+        taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew));
+        uv_mutex_unlock(&global.udfsMutex);
       }
 
-            char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
+      uv_mutex_lock(&udf->lock);
-            strcpy(normalFuncName, setup->udfName);
+      if (udf->state == UDF_STATE_INIT) {
-	    //TODO error, multi-thread, same udf, lock it
+        udf->state = UDF_STATE_LOADING;
-	    //TODO find all functions normal, init, destroy, normal, merge, finalize
+        udfdLoadUdf(setup->udfName, udf);
-            uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->scalarProcFunc));
+        udf->state = UDF_STATE_READY;
-            char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
+        uv_cond_broadcast(&udf->condReady);
-            char *freeSuffix = "_free";
+        uv_mutex_unlock(&udf->lock);
-            strncpy(freeFuncName, normalFuncName, strlen(normalFuncName));
+      } else {
-            strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
+        while (udf->state != UDF_STATE_READY) {
-            uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
+          uv_cond_wait(&udf->condReady, &udf->lock);
-
+        }
+        uv_mutex_unlock(&udf->lock);
+      }
       SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle));
       handle->udf = udf;
-            udf->refCount++;
       // TODO: allocate private structure and call init function and set it to handle
       SUdfResponse rsp;
       rsp.seqNum = request.seqNum;
@@ -112,8 +164,8 @@ void udfdProcessRequest(uv_work_t *req) {
     }
 
     case UDF_TASK_CALL: {
-            debugPrint("%s", "process call request");
       SUdfCallRequest *call = &request.call;
+      fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle);
       SUdfHandle      *handle = (SUdfHandle *)(call->udfHandle);
       SUdf            *udf = handle->udf;
 
@@ -143,20 +195,28 @@ void udfdProcessRequest(uv_work_t *req) {
       encodeUdfResponse(&buf, rsp);
       uvUdf->output = uv_buf_init(bufBegin, len);
 
-            //TODO: free
+      // TODO: free udf column
       udf->freeUdfColumn(&output);
 
       taosMemoryFree(uvUdf->input.base);
       break;
     }
     case UDF_TASK_TEARDOWN: {
-            debugPrint("%s", "process teardown request");
-
       SUdfTeardownRequest *teardown = &request.teardown;
+      fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle)
       SUdfHandle          *handle = (SUdfHandle *)(teardown->udfHandle);
       SUdf                *udf = handle->udf;
+      bool unloadUdf = false;
+      uv_mutex_lock(&global.udfsMutex);
       udf->refCount--;
       if (udf->refCount == 0) {
+        unloadUdf = true;
+        taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN);
+      }
+      uv_mutex_unlock(&global.udfsMutex);
+      if (unloadUdf) {
+        uv_cond_destroy(&udf->condReady);
+        uv_mutex_destroy(&udf->lock);
         uv_dlclose(&udf->lib);
         taosMemoryFree(udf);
       }
@@ -181,26 +241,21 @@ void udfdProcessRequest(uv_work_t *req) {
     default: {
       break;
     }
-
   }
-
 }
 
 void udfdOnWrite(uv_write_t *req, int status) {
-    debugPrint("%s", "server after writing to pipe");
-    if (status < 0) {
-        debugPrint("Write error %s", uv_err_name(status));
-    }
   SUvUdfWork *work = (SUvUdfWork *)req->data;
-    debugPrint("\tlength: %zu", work->output.len);
+  if (status < 0) {
+    //TODO:log error and process it.
+  }
+  fnDebug("send response. length:%zu, status: %s", work->output.len, uv_err_name(status));
   taosMemoryFree(work->output.base);
   taosMemoryFree(work);
   taosMemoryFree(req);
 }
 
-
 void udfdSendResponse(uv_work_t *work, int status) {
-    debugPrint("%s", "send response");
   SUvUdfWork *udfWork = (SUvUdfWork *)(work->data);
 
   uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t));
@@ -211,7 +266,6 @@ void udfdSendResponse(uv_work_t *work, int status) {
 }
 
 void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
-    debugPrint("%s", "server allocate buffer for read");
   SUdfdUvConn *ctx = handle->data;
   int32_t      msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
   if (ctx->inputCap == 0) {
@@ -241,8 +295,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
       buf->len = 0;
     }
   }
-    debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal);
+  fnDebug("allocate buf. input buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal);
-
 }
 
 bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) {
@@ -250,6 +303,7 @@ bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) {
     pipe->inputTotal = *(int32_t *)(pipe->inputBuf);
   }
   if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) {
+    fnDebug("receive request complete. length %d", pipe->inputLen);
     return true;
   }
   return false;
@@ -265,7 +319,7 @@ void udfdHandleRequest(SUdfdUvConn *conn) {
   conn->inputCap = 0;
   conn->inputTotal = -1;
   work->data = udfWork;
-    uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse);
+  uv_queue_work(global.loop, work, udfdProcessRequest, udfdSendResponse);
 }
 
 void udfdPipeCloseCb(uv_handle_t *pipe) {
@@ -275,13 +329,10 @@ void udfdPipeCloseCb(uv_handle_t *pipe) {
   taosMemoryFree(conn);
 }
 
-void udfdUvHandleError(SUdfdUvConn *conn) {
+void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); }
-    uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb);
-}
 
 void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
-    debugPrint("%s, nread: %zd", "read from pipe", nread);
+  fnDebug("udf read %zu bytes from client", nread);
-
   if (nread == 0) return;
 
   SUdfdUvConn *conn = client->data;
@@ -297,7 +348,7 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
   }
 
   if (nread < 0) {
-        debugPrint("Read error %s", uv_err_name(nread));
+    fnDebug("Receive error %s", uv_err_name(nread));
     if (nread == UV_EOF) {
       // TODO check more when close
     } else {
@@ -307,14 +358,14 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
 }
 
 void udfdOnNewConnection(uv_stream_t *server, int status) {
-    debugPrint("%s", "on new connection");
+  fnDebug("new connection");
   if (status < 0) {
     // TODO
     return;
   }
 
   uv_pipe_t *client = (uv_pipe_t *)taosMemoryMalloc(sizeof(uv_pipe_t));
-    uv_pipe_init(loop, client, 0);
+  uv_pipe_init(global.loop, client, 0);
   if (uv_accept(server, (uv_stream_t *)client) == 0) {
     SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn));
     ctx->client = (uv_stream_t *)client;
@@ -331,27 +382,17 @@ void udfdOnNewConnection(uv_stream_t *server, int status) {
 
 void removeListeningPipe(int sig) {
   uv_fs_t req;
-    uv_fs_unlink(loop, &req, "udf.sock", NULL);
+  uv_fs_unlink(global.loop, &req, "udf.sock", NULL);
   exit(0);
 }
 
-typedef struct SServerContext {
+void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
-  void *clientRpc;
-} SUdfdContext;
 
-
+int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char* udfName, SUdf* udf) {
-void udfdProcessRpcRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)  {
-
-  return;
-}
-
-int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int32_t numOfUdfs) {
   SRetrieveFuncReq retrieveReq = {0};
   retrieveReq.numOfFuncs = 1;
   retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
-  for (int32_t i = 0; i < numOfUdfs; ++i) {
+  taosArrayPush(retrieveReq.pFuncNames, udfName);
-    taosArrayPush(retrieveReq.pFuncNames, udfNames[i]);
-  }
 
   int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
   void   *pReq = rpcMallocCont(contLen);
@@ -370,13 +411,20 @@ int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int3
 
   SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
 
+  char path[PATH_MAX] = {0};
+  taosGetTmpfilePath("/tmp", "libudf", path);
+  TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  // TODO check for failure of flush to disk
+  taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
+  taosCloseFile(&file);
+  strncpy(udf->path, path, strlen(path));
   taosArrayDestroy(retrieveRsp.pFuncInfos);
 
   rpcFreeCont(rpcRsp.pCont);
   return 0;
 }
 
-int32_t openUdfdClientRpc(SUdfdContext *ctx) {
+int32_t udfdOpenClientRpc() {
   char *pass = "taosdata";
   char *user = "root";
   char  secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
@@ -388,46 +436,149 @@ int32_t openUdfdClientRpc(SUdfdContext *ctx) {
   rpcInit.sessions = 1024;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.idleTime = 30 * 1000;
-  rpcInit.parent = ctx;
+  rpcInit.parent = &global;
 
   rpcInit.user = (char *)user;
   rpcInit.ckey = (char *)"key";
   rpcInit.secret = (char *)secretEncrypt;
   rpcInit.spi = 1;
 
-  ctx->clientRpc = rpcOpen(&rpcInit);
+  global.clientRpc = rpcOpen(&rpcInit);
 
   return 0;
 }
 
-int32_t closeUdfdClientRpc(SUdfdContext *ctx) {
+int32_t udfdCloseClientRpc() {
-  rpcClose(ctx->clientRpc);
+  rpcClose(global.clientRpc);
+  return 0;
+}
+
+static void udfdPrintVersion() {
+#ifdef TD_ENTERPRISE
+  char *releaseName = "enterprise";
+#else
+  char *releaseName = "community";
+#endif
+  printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
+  printf("gitinfo: %s\n", gitinfo);
+  printf("buildInfo: %s\n", buildinfo);
+}
+
+static int32_t udfdParseArgs(int32_t argc, char *argv[]) {
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("config file path overflow");
+          return -1;
+        }
+        tstrncpy(configDir, argv[i], PATH_MAX);
+      } else {
+        printf("'-c' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-V") == 0) {
+      global.printVersion = true;
+    } else {
+    }
+  }
 
   return 0;
 }
 
-int main() {
+static int32_t udfdInitLog() {
-    debugPrint("libuv version: %x", UV_VERSION_HEX);
+  char logName[12] = {0};
+  snprintf(logName, sizeof(logName), "%slog", "udfd");
+  return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
+}
+
+static int32_t udfdUvInit() {
+  uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t));
+  if (loop) {
+    uv_loop_init(loop);
+  }
+  global.loop = loop;
+  char dnodeId[8] = {0};
+  size_t dnodeIdSize;
+  uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
+  char listenPipeName[32] = {0};
+  snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
+  strcpy(global.listenPipeName, listenPipeName);
 
-    loop = uv_default_loop();
   uv_fs_t req;
-    uv_fs_unlink(loop, &req, "udf.sock", NULL);
+  uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
 
   uv_pipe_t server;
-    uv_pipe_init(loop, &server, 0);
+  uv_pipe_init(global.loop, &server, 0);
 
   signal(SIGINT, removeListeningPipe);
 
   int r;
-    if ((r = uv_pipe_bind(&server, "udf.sock"))) {
+  fnInfo("bind to pipe %s", global.listenPipeName);
-        debugPrint("Bind error %s\n", uv_err_name(r));
+  if ((r = uv_pipe_bind(&server, listenPipeName))) {
+    fnError("Bind error %s", uv_err_name(r));
     removeListeningPipe(0);
-        return 1;
+    return -1;
   }
   if ((r = uv_listen((uv_stream_t *)&server, 128, udfdOnNewConnection))) {
-        debugPrint("Listen error %s", uv_err_name(r));
+    fnError("Listen error %s", uv_err_name(r));
-        return 2;
+    removeListeningPipe(0);
+    return -2;
   }
-    uv_run(loop, UV_RUN_DEFAULT);
+  return 0;
-    uv_loop_close(loop);
+}
+
+static int32_t udfdRun() {
+  global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  uv_mutex_init(&global.udfsMutex);
+
+  //TOOD: client rpc to fetch udf function info from mnode
+  if (udfdOpenClientRpc() != 0) {
+    fnError("open rpc connection to mnode failure");
+    return -1;
+  }
+
+  if (udfdUvInit() != 0) {
+    fnError("uv init failure");
+    return -2;
+  }
+
+  fnInfo("start the udfd");
+  int code = uv_run(global.loop, UV_RUN_DEFAULT);
+  fnInfo("udfd stopped. result: %s", uv_err_name(code));
+  int codeClose = uv_loop_close(global.loop);
+  fnDebug("uv loop close. result: %s", uv_err_name(codeClose));
+  udfdCloseClientRpc();
+  uv_mutex_destroy(&global.udfsMutex);
+  taosHashCleanup(global.udfsHash);
+  return code;
+}
+
+int main(int argc, char* argv[]) {
+  if (!taosCheckSystemIsSmallEnd()) {
+    printf("failed to start since on non-small-end machines\n");
+    return -1;
+  }
+
+  if (udfdParseArgs(argc, argv) != 0) {
+    printf("failed to start since parse args error\n");
+    return -1;
+  }
+
+  if (global.printVersion) {
+    udfdPrintVersion();
+    return 0;
+  }
+
+  if (udfdInitLog() != 0) {
+    printf("failed to start since init log error\n");
+    return -1;
+  }
+
+  if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) {
+    fnError("failed to start since read config error");
+    return -1;
+  }
+
+  return udfdRun();
 }

source/libs/function/test/runUdf.c

@@ -8,7 +8,7 @@
 #include "tdatablock.h"
 
 int main(int argc, char *argv[]) {
-    startUdfService();
+  createUdfdProxy(1);
     uv_sleep(1000);
     char path[256] = {0};
     size_t cwdSize = 256;
@@ -53,5 +53,5 @@ int main(int argc, char *argv[]) {
     }
     teardownUdf(handle);
 
-    stopUdfService();
+    destroyUdfdProxy(1);
 }

source/os/src/osFile.c

@@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&(pFile->rwlock));
 #endif
-  assert(pFile->fd >= 0); // Please check if you have closed the file.
+  /*assert(pFile->fd >= 0); // Please check if you have closed the file.*/
 
   int64_t nleft = count;
   int64_t nwritten = 0;

source/util/src/tarray.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "tarray.h"
+#include "tcoding.h"
 
 SArray* taosArrayInit(size_t size, size_t elemSize) {
   assert(elemSize > 0);
@@ -75,7 +76,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
 }
 
 void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) {
-  if (pArray == NULL || pData == NULL) {
+  if (pData == NULL) {
     return NULL;
   }
 
@@ -317,7 +318,6 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) {
   pArray->size = 0;
 }
 
-
 void* taosArrayDestroy(SArray* pArray) {
   if (pArray) {
     taosMemoryFree(pArray->pData);
@@ -327,7 +327,14 @@ void* taosArrayDestroy(SArray* pArray) {
   return NULL;
 }
 
-void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) {
+void taosArrayDestroyP(SArray* pArray, FDelete fp) {
+  for (int32_t i = 0; i < pArray->size; i++) {
+    fp(*(void**)TARRAY_GET_ELEM(pArray, i));
+  }
+  taosArrayDestroy(pArray);
+}
+
+void taosArrayDestroyEx(SArray* pArray, FDelete fp) {
   if (pArray == NULL) {
     return;
   }
@@ -436,6 +443,39 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void
   return;
 }
 
+SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) {
+  ASSERT(pSrc->elemSize == sizeof(void*));
+  SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*));
+  for (int32_t i = 0; i < pSrc->size; i++) {
+    void* clone = deepCopy(taosArrayGetP(pSrc, i));
+    taosArrayPush(pArray, &clone);
+  }
+  return pArray;
+}
+
+int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) {
+  int32_t tlen = 0;
+  int32_t sz = pArray->size;
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    void* data = taosArrayGetP(pArray, i);
+    tlen += encode(buf, data);
+  }
+  return tlen;
+}
+
+void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz) {
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  *pArray = taosArrayInit(sz, sizeof(void*));
+  for (int32_t i = 0; i < sz; i++) {
+    void* data = taosMemoryCalloc(1, dataSz);
+    buf = decode(buf, data);
+    taosArrayPush(*pArray, &data);
+  }
+  return (void*)buf;
+}
+
 // order array<type *>
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) {
   taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param);

source/util/src/tlog.c

@@ -92,6 +92,7 @@ int32_t tsdbDebugFlag = 131;
 int32_t tqDebugFlag = 135;
 int32_t fsDebugFlag = 135;
 int32_t metaDebugFlag = 135;
+int32_t fnDebugFlag = 135;
 
 int64_t dbgEmptyW = 0;
 int64_t dbgWN = 0;
@@ -753,6 +754,7 @@ void taosSetAllDebugFlag(int32_t flag) {
   tsdbDebugFlag = flag;
   tqDebugFlag = flag;
   fsDebugFlag = flag;
+  fnDebugFlag = flag;
 
   uInfo("all debug flag are set to %d", flag);
 }

tests/script/tsim/tmq/basic1.sim

@@ -33,8 +33,8 @@ endi
 sql connect
 
 $dbNamme = d0
-print =============== create database , vgroup 1
+print =============== create database , vgroup 4
-sql create database $dbNamme vgroups 1
+sql create database $dbNamme vgroups 4
 sql use $dbNamme
 
 print =============== create super table