From 416c87b5b7bbbf2b5b553aa5b578b11fd711cfa8 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 23 Jan 2024 18:48:19 +0800
Subject: [PATCH 01/33] fix:[TD-28446] modify trans confilct in subscribe

---
 source/dnode/mnode/impl/src/mndConsumer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 4db000287c..f2363a588f 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -525,7 +525,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
   }
 
   // check topic existence
-  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
+  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "subscribe");
   if (pTrans == NULL) {
     goto _over;
   }

From 2798ff824cf4e56eb9ef86377e36a19d5a70e169 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 25 Jan 2024 16:07:09 +0800
Subject: [PATCH 02/33] feat:[TD-28247]add grant for subscribe and stream

---
 include/common/tgrant.h                       |   1 +
 include/common/tmsg.h                         |  24 ++-
 source/client/src/clientTmq.c                 |  33 +++-
 source/common/src/tmsg.c                      |  83 ++++++----
 source/dnode/mnode/impl/inc/mndPrivilege.h    |   1 -
 source/dnode/mnode/impl/src/mndConsumer.c     |  59 +++++++-
 source/dnode/mnode/impl/src/mndPrivilege.c    |   3 -
 source/dnode/mnode/impl/src/mndStream.c       |  67 +++++++--
 source/dnode/mnode/impl/src/mndUser.c         |   2 +
 .../0-others/subscribe_stream_privilege.py    | 142 ++++++++++++++++++
 10 files changed, 351 insertions(+), 64 deletions(-)
 create mode 100644 tests/system-test/0-others/subscribe_stream_privilege.py

diff --git a/include/common/tgrant.h b/include/common/tgrant.h
index f06fca8014..cfb1401698 100644
--- a/include/common/tgrant.h
+++ b/include/common/tgrant.h
@@ -48,6 +48,7 @@ typedef enum {
   TSDB_GRANT_CPU_CORES,
   TSDB_GRANT_STABLE,
   TSDB_GRANT_TABLE,
+  TSDB_GRANT_SUBSCRIBE,
 } EGrantType;
 
 int32_t grantCheck(EGrantType grant);
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 2ee48a18e0..05e2738f7c 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3360,6 +3360,7 @@ typedef struct {
   char   name[TSDB_STREAM_FNAME_LEN];
   int8_t igNotExists;
   int8_t igUntreated;
+  int8_t suspend;
 } SMResumeStreamReq;
 
 int32_t tSerializeSMResumeStreamReq(void* buf, int32_t bufLen, const SMResumeStreamReq* pReq);
@@ -3754,7 +3755,12 @@ typedef struct {
 } SMqHbReq;
 
 typedef struct {
-  int8_t reserved;
+  char           topic[TSDB_TOPIC_FNAME_LEN];
+  int8_t         noPrivilege;
+} STopicPrivilege;
+
+typedef struct {
+  SArray* topicPrivileges;   // SArray<STopicPrivilege>
 } SMqHbRsp;
 
 typedef struct {
@@ -3773,18 +3779,6 @@ typedef struct {
   SVCreateTbReq cTbReq;
 } SVSubmitBlk;
 
-typedef struct {
-  int32_t flags;
-  int32_t nBlocks;
-  union {
-    SArray*      pArray;
-    SVSubmitBlk* pBlocks;
-  };
-} SVSubmitReq;
-
-int32_t tEncodeSVSubmitReq(SEncoder* pCoder, const SVSubmitReq* pReq);
-int32_t tDecodeSVSubmitReq(SDecoder* pCoder, SVSubmitReq* pReq);
-
 typedef struct {
   SMsgHead header;
   uint64_t sId;
@@ -3893,6 +3887,10 @@ int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq);
 int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq);
 int32_t tDeatroySMqHbReq(SMqHbReq* pReq);
 
+int32_t tSerializeSMqHbRsp(void* buf, int32_t bufLen, SMqHbRsp* pRsp);
+int32_t tDeserializeSMqHbRsp(void* buf, int32_t bufLen, SMqHbRsp* pRsp);
+int32_t tDeatroySMqHbRsp(SMqHbRsp* pRsp);
+
 int32_t tSerializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
 int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
 
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 69681b9ae0..79611b7eee 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -155,6 +155,7 @@ typedef struct {
   char           db[TSDB_DB_FNAME_LEN];
   SArray*        vgs;  // SArray<SMqClientVg>
   SSchemaWrapper schema;
+  int8_t         noPrivilege;
 } SMqClientTopic;
 
 typedef struct {
@@ -739,6 +740,29 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
 
 int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
   if (pMsg) {
+    SMqHbRsp rsp = {0};
+    tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp);
+
+    int64_t refId = *(int64_t*)param;
+    tmq_t*  tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
+    if (tmq != NULL) {
+      taosWLockLatch(&tmq->lock);
+      for(int32_t i = 0; i < taosArrayGetSize(rsp.topicPrivileges); i++){
+        STopicPrivilege* privilege = taosArrayGet(rsp.topicPrivileges, i);
+        if(privilege->noPrivilege == 1){
+          int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
+          for (int32_t j = 0; j < topicNumCur; j++) {
+            SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j);
+            if(strcmp(pTopicCur->topicName, privilege->topic) == 0){
+              tscInfo("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic);
+              pTopicCur->noPrivilege = 1;
+            }
+          }
+        }
+      }
+      taosWUnLockLatch(&tmq->lock);
+    }
+
     taosMemoryFree(pMsg->pData);
     taosMemoryFree(pMsg->pEpSet);
   }
@@ -809,7 +833,9 @@ void tmqSendHbReq(void* param, void* tmrId) {
 
   sendInfo->requestId = generateRequestId();
   sendInfo->requestObjRefId = 0;
-  sendInfo->param = NULL;
+  sendInfo->paramFreeFp = taosMemoryFree;
+  sendInfo->param = taosMemoryMalloc(sizeof(int64_t));
+  *(int64_t *)sendInfo->param = refId;
   sendInfo->fp = tmqHbCb;
   sendInfo->msgType = TDMT_MND_TMQ_HB;
 
@@ -1705,7 +1731,10 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
   for (int i = 0; i < numOfTopics; i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
     int32_t         numOfVg = taosArrayGetSize(pTopic->vgs);
-
+    if(pTopic->noPrivilege){
+      tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName);
+      continue;
+    }
     for (int j = 0; j < numOfVg; j++) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) {  // less than 10ms
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index c9e2908e8a..a395884538 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -6139,6 +6139,55 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) {
   return 0;
 }
 
+int32_t tDeatroySMqHbRsp(SMqHbRsp *pRsp) {
+  taosArrayDestroy(pRsp->topicPrivileges);
+  return 0;
+}
+
+int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  int32_t sz = taosArrayGetSize(pRsp->topicPrivileges);
+  if (tEncodeI32(&encoder, sz) < 0) return -1;
+  for (int32_t i = 0; i < sz; ++i) {
+    STopicPrivilege *privilege = (STopicPrivilege *)taosArrayGet(pRsp->topicPrivileges, i);
+    if (tEncodeCStr(&encoder, privilege->topic) < 0) return -1;
+    if (tEncodeI8(&encoder, privilege->noPrivilege) < 0) return -1;
+  }
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+
+  return tlen;
+}
+
+int32_t tDeserializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, (char *)buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  int32_t sz = 0;
+  if (tDecodeI32(&decoder, &sz) < 0) return -1;
+  if (sz > 0) {
+    pRsp->topicPrivileges = taosArrayInit(sz, sizeof(STopicPrivilege));
+    if (NULL == pRsp->topicPrivileges) return -1;
+    for (int32_t i = 0; i < sz; ++i) {
+      STopicPrivilege *data = taosArrayReserve(pRsp->topicPrivileges, 1);
+      if (tDecodeCStrTo(&decoder, data->topic) < 0) return -1;
+      if (tDecodeI8(&decoder, &data->noPrivilege) < 0) return -1;
+    }
+  }
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
 int32_t tDeatroySMqHbReq(SMqHbReq *pReq) {
   for (int i = 0; i < taosArrayGetSize(pReq->topics); i++) {
     TopicOffsetRows *vgs = taosArrayGet(pReq->topics, i);
@@ -6194,7 +6243,7 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) {
     if (NULL == pReq->topics) return -1;
     for (int32_t i = 0; i < sz; ++i) {
       TopicOffsetRows *data = taosArrayReserve(pReq->topics, 1);
-      tDecodeCStrTo(&decoder, data->topicName);
+      if (tDecodeCStrTo(&decoder, data->topicName) < 0) return -1;
       int32_t szVgs = 0;
       if (tDecodeI32(&decoder, &szVgs) < 0) return -1;
       if (szVgs > 0) {
@@ -7753,36 +7802,6 @@ static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t
   return 0;
 }
 
-int32_t tEncodeSVSubmitReq(SEncoder *pCoder, const SVSubmitReq *pReq) {
-  int32_t nBlocks = taosArrayGetSize(pReq->pArray);
-
-  if (tStartEncode(pCoder) < 0) return -1;
-
-  if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1;
-  if (tEncodeI32v(pCoder, nBlocks) < 0) return -1;
-  for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
-    if (tEncodeSVSubmitBlk(pCoder, (SVSubmitBlk *)taosArrayGet(pReq->pArray, iBlock), pReq->flags) < 0) return -1;
-  }
-
-  tEndEncode(pCoder);
-  return 0;
-}
-
-int32_t tDecodeSVSubmitReq(SDecoder *pCoder, SVSubmitReq *pReq) {
-  if (tStartDecode(pCoder) < 0) return -1;
-
-  if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1;
-  if (tDecodeI32v(pCoder, &pReq->nBlocks) < 0) return -1;
-  pReq->pBlocks = tDecoderMalloc(pCoder, sizeof(SVSubmitBlk) * pReq->nBlocks);
-  if (pReq->pBlocks == NULL) return -1;
-  for (int32_t iBlock = 0; iBlock < pReq->nBlocks; iBlock++) {
-    if (tDecodeSVSubmitBlk(pCoder, pReq->pBlocks + iBlock, pReq->flags) < 0) return -1;
-  }
-
-  tEndDecode(pCoder);
-  return 0;
-}
-
 static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) {
   if (tStartEncode(pEncoder) < 0) return -1;
 
@@ -8914,6 +8933,7 @@ int32_t tSerializeSMResumeStreamReq(void *buf, int32_t bufLen, const SMResumeStr
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igUntreated) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->suspend) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -8928,6 +8948,7 @@ int32_t tDeserializeSMResumeStreamReq(void *buf, int32_t bufLen, SMResumeStreamR
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igUntreated) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->suspend) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h
index 4a8fb20715..6f74ea3b36 100644
--- a/source/dnode/mnode/impl/inc/mndPrivilege.h
+++ b/source/dnode/mnode/impl/inc/mndPrivilege.h
@@ -30,7 +30,6 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType
 int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
 int32_t mndCheckViewPrivilege(SMnode *pMnode, const char *user, EOperType operType, const char *pViewFName);
 int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic);
-int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName);
 int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
 int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
 int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp);
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 4db000287c..14e3df2af9 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -102,7 +102,8 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
     }
 
     if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) {
-      code = -1;
+      code = TSDB_CODE_MND_NO_RIGHTS;
+      terrno = TSDB_CODE_MND_NO_RIGHTS;
       goto FAILED;
     }
 
@@ -220,22 +221,52 @@ FAIL:
   return -1;
 }
 
+static int32_t checkPrivilege(SMnode  *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char* user){
+  rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege));
+  if(rsp->topicPrivileges == NULL){
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return terrno;
+  }
+  for(int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++){
+    char *topic = taosArrayGetP(pConsumer->currentTopics, i);
+    SMqTopicObj* pTopic = mndAcquireTopic(pMnode, topic);
+    if (pTopic == NULL) {  // terrno has been set by callee function
+      continue;
+    }
+    STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1);
+    if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIBE) < 0) {
+      data->noPrivilege = 1;
+    } else{
+      data->noPrivilege = 0;
+    }
+    mndReleaseTopic(pMnode, pTopic);
+  }
+  return 0;
+}
+
 static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
   int32_t code = 0;
   SMnode  *pMnode = pMsg->info.node;
   SMqHbReq req = {0};
+  SMqHbRsp rsp = {0};
+  SMqConsumerObj *pConsumer = NULL;
 
-  if ((code = tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req)) < 0) {
+  if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = TSDB_CODE_OUT_OF_MEMORY;
     goto end;
   }
 
   int64_t         consumerId = req.consumerId;
-  SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
+  pConsumer = mndAcquireConsumer(pMnode, consumerId);
   if (pConsumer == NULL) {
     mError("consumer:0x%" PRIx64 " not exist", consumerId);
     terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
-    code = -1;
+    code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
+    goto end;
+  }
+  code = checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user);
+  if(code != 0){
     goto end;
   }
 
@@ -280,9 +311,22 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
     mndReleaseSubscribe(pMnode, pSub);
   }
 
-  mndReleaseConsumer(pMnode, pConsumer);
+  // encode rsp
+  int32_t tlen = tSerializeSMqHbRsp(NULL, 0, &rsp);
+  void   *buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+
+  tSerializeSMqHbRsp(&buf, tlen, &rsp);
+  pMsg->info.rsp = buf;
+  pMsg->info.rspLen = tlen;
 
 end:
+  tDeatroySMqHbRsp(&rsp);
+  mndReleaseConsumer(pMnode, pConsumer);
   tDeatroySMqHbReq(&req);
   return code;
 }
@@ -500,6 +544,11 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
   SMnode *pMnode = pMsg->info.node;
   char   *msgStr = pMsg->pCont;
 
+  if(grantCheck(TSDB_GRANT_SUBSCRIBE) < 0){
+    terrno = TSDB_CODE_GRANT_EXPIRED;
+    return -1;
+  }
+
   SCMSubscribeReq subscribe = {0};
   tDeserializeSCMSubscribeReq(msgStr, &subscribe);
 
diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c
index d4c0a6b36b..13a80cb1a6 100644
--- a/source/dnode/mnode/impl/src/mndPrivilege.c
+++ b/source/dnode/mnode/impl/src/mndPrivilege.c
@@ -30,9 +30,6 @@ int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType op
 }
 
 int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; }
-int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName) {
-  return 0;
-}
 
 
 int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteListRsp *pWhiteListRsp) {
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 441305f282..bf92a51ed8 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -808,6 +808,11 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
   int32_t     sqlLen = 0;
   terrno = TSDB_CODE_SUCCESS;
 
+  if(grantCheck(TSDB_GRANT_STREAMS) < 0){
+    terrno = TSDB_CODE_GRANT_STREAM_LIMITED;
+    return -1;
+  }
+
   SCMCreateStreamReq createStreamReq = {0};
   if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createStreamReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -2034,17 +2039,22 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
   SMnode     *pMnode = pReq->info.node;
   SStreamObj *pStream = NULL;
 
-  SMResumeStreamReq pauseReq = {0};
-  if (tDeserializeSMResumeStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) {
+  if(grantCheck(TSDB_GRANT_STREAMS) < 0){
+    terrno = TSDB_CODE_GRANT_EXPIRED;
+    return -1;
+  }
+
+  SMResumeStreamReq resumeReq = {0};
+  if (tDeserializeSMResumeStreamReq(pReq->pCont, pReq->contLen, &resumeReq) < 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
   }
 
-  pStream = mndAcquireStream(pMnode, pauseReq.name);
+  pStream = mndAcquireStream(pMnode, resumeReq.name);
 
   if (pStream == NULL) {
-    if (pauseReq.igNotExists) {
-      mInfo("stream:%s, not exist, if exist is set", pauseReq.name);
+    if (resumeReq.igNotExists) {
+      mInfo("stream:%s, not exist, if exist is set", resumeReq.name);
       sdbRelease(pMnode->pSdb, pStream);
       return 0;
     } else {
@@ -2072,12 +2082,12 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, MND_STREAM_RESUME_NAME);
   if (pTrans == NULL) {
-    mError("stream:%s, failed to resume stream since %s", pauseReq.name, terrstr());
+    mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr());
     sdbRelease(pMnode->pSdb, pStream);
     return -1;
   }
 
-  mInfo("trans:%d used to resume stream:%s", pTrans->id, pauseReq.name);
+  mInfo("trans:%d used to resume stream:%s", pTrans->id, resumeReq.name);
 
   mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) {
@@ -2089,8 +2099,8 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
   int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid);
 
   // resume all tasks
-  if (mndResumeAllStreamTasks(pTrans, pMnode, pStream, pauseReq.igUntreated) < 0) {
-    mError("stream:%s, failed to drop task since %s", pauseReq.name, terrstr());
+  if (mndResumeAllStreamTasks(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) {
+    mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr());
     sdbRelease(pMnode->pSdb, pStream);
     mndTransDrop(pTrans);
     return -1;
@@ -3030,6 +3040,39 @@ static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) {
   }
 }
 
+int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){
+  SSdb       *pSdb = pMnode->pSdb;
+  SStreamObj *pStream = NULL;
+  void* pIter = NULL;
+  while(1) {
+    pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
+    if (pIter == NULL) break;
+
+    if(pStream->status != STREAM_STATUS__PAUSE){
+      SMPauseStreamReq *reqPause = rpcMallocCont(sizeof(SMPauseStreamReq));
+      if (reqPause == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        sdbRelease(pSdb, pStream);
+        return -1;
+      }
+      strcpy(reqPause->name, pStream->name);
+      reqPause->igNotExists = 1;
+
+      SRpcMsg rpcMsg = {
+          .msgType = TDMT_MND_PAUSE_STREAM,
+          .pCont = reqPause,
+          .contLen = sizeof(SMPauseStreamReq),
+          .info = *info,
+      };
+
+      tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+    }
+
+    sdbRelease(pSdb, pStream);
+  }
+  return 0;
+}
+
 int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   SMnode      *pMnode = pReq->info.node;
   SStreamHbMsg req = {0};
@@ -3039,6 +3082,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   int64_t streamId = 0;
   int32_t transId = 0;
 
+  if(grantCheck(TSDB_GRANT_STREAMS) < 0){
+    if(suspendAllStreams(pMnode, &pReq->info) < 0){
+      return -1;
+    }
+  }
+
   SDecoder decoder = {0};
   tDecoderInit(&decoder, pReq->pCont, pReq->contLen);
 
diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c
index 0e3b544508..5e5a3626a4 100644
--- a/source/dnode/mnode/impl/src/mndUser.c
+++ b/source/dnode/mnode/impl/src/mndUser.c
@@ -1925,6 +1925,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
       return -1;
     }
     taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
+    mndReleaseTopic(pMnode, pTopic);
   }
 
   if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
@@ -1935,6 +1936,7 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
       return -1;
     }
     taosHashRemove(pNewUser->topics, pAlterReq->objname, len);
+    mndReleaseTopic(pMnode, pTopic);
   }
 
   return TSDB_CODE_SUCCESS;
diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py
new file mode 100644
index 0000000000..881060ee3c
--- /dev/null
+++ b/tests/system-test/0-others/subscribe_stream_privilege.py
@@ -0,0 +1,142 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+import time
+
+import taos
+from taos.tmq import *
+from util.cases import *
+from util.common import *
+from util.log import *
+from util.sql import *
+from util.sqlset import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        self.setsql = TDSetSql()
+        self.stbname = 'stb'
+        self.user_name = 'test'
+        self.binary_length = 20  # the length of binary for column_dict
+        self.nchar_length = 20  # the length of nchar for column_dict
+        self.dbnames = ['db1']
+        self.column_dict = {
+            'ts': 'timestamp',
+            'col1': 'float',
+            'col2': 'int',
+            'col3': 'float',
+        }
+        
+        self.tag_dict = {
+            't1': 'int',
+            't2': f'binary({self.binary_length})'
+        }
+        
+        self.tag_list = [
+            f'1, "Beijing"',
+            f'2, "Shanghai"',
+            f'3, "Guangzhou"',
+            f'4, "Shenzhen"'
+        ]
+        
+        self.values_list = [
+            f'now, 9.1, 200, 0.3'            
+        ]
+        
+        self.tbnum = 4
+        self.topic_name = 'topic1'
+
+
+    def prepare_data(self):
+        for db in self.dbnames:
+            tdSql.execute(f"create database {db}")
+            tdSql.execute(f"use {db}")
+            tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict))
+            for i in range(self.tbnum):
+                tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})')
+                for j in self.values_list:
+                    tdSql.execute(f'insert into {self.stbname}_{i} values({j})')
+
+    def consumeTest(self):
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": self.user_name,
+            "td.connect.pass": "test",
+            "auto.offset.reset": "earliest"
+        }
+        consumer = Consumer(consumer_dict)
+
+        tdLog.debug("test subscribe topic created by other user")
+        exceptOccured = False
+        try:
+            consumer.subscribe([self.topic_name])
+        except TmqError:
+            exceptOccured = True
+
+        if not exceptOccured:
+            tdLog.exit(f"has no privilege, should except")
+
+        tdLog.debug("test subscribe topic privilege granted by other user")
+        tdSql.execute(f'grant subscribe on {self.topic_name} to {self.user_name}')
+
+        exceptOccured = False
+        try:
+            consumer.subscribe([self.topic_name])
+        except TmqError:
+            exceptOccured = True
+
+        if exceptOccured:
+            tdLog.exit(f"has privilege, should not except")
+
+        cnt = 0
+        try:
+            while True:
+                res = consumer.poll(1)
+                cnt += 1
+                if cnt == 1:
+                    if not res:
+                        tdLog.exit(f"grant privilege, should get res")
+                elif cnt == 2:
+                    if res:
+                        time.sleep(1000)
+                        tdLog.exit(f"revoke privilege, should get NULL")
+
+                else:
+                    break
+
+                tdLog.debug("test subscribe topic privilege revoked by other user")
+                tdSql.execute(f'revoke subscribe on {self.topic_name} from {self.user_name}')
+                time.sleep(5)
+
+        finally:
+            consumer.close()
+        time.sleep(1000)
+
+    def create_user(self):
+        tdSql.execute(f'create topic {self.topic_name} as database {self.dbnames[0]}')
+        tdSql.execute(f'create user {self.user_name} pass "test"')
+
+    def run(self):
+        self.prepare_data()
+        self.create_user()
+        self.consumeTest()
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

From 474514ab6682e8a6b276b983cf21836c641b616e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 25 Jan 2024 17:29:55 +0800
Subject: [PATCH 03/33] fix:test case error

---
 source/dnode/mnode/impl/src/mndConsumer.c             |  3 ++-
 .../0-others/subscribe_stream_privilege.py            | 11 +++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 14e3df2af9..2c8a193121 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -234,6 +234,7 @@ static int32_t checkPrivilege(SMnode  *pMnode, SMqConsumerObj *pConsumer, SMqHbR
       continue;
     }
     STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1);
+    strcpy(data->topic, topic);
     if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIBE) < 0) {
       data->noPrivilege = 1;
     } else{
@@ -320,7 +321,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
     goto end;
   }
 
-  tSerializeSMqHbRsp(&buf, tlen, &rsp);
+  tSerializeSMqHbRsp(buf, tlen, &rsp);
   pMsg->info.rsp = buf;
   pMsg->info.rspLen = tlen;
 
diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py
index 881060ee3c..5f40450af4 100644
--- a/tests/system-test/0-others/subscribe_stream_privilege.py
+++ b/tests/system-test/0-others/subscribe_stream_privilege.py
@@ -22,6 +22,8 @@ from util.sqlset import *
 
 
 class TDTestCase:
+    clientCfgDict = {'debugFlag': 135}
+    updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict}
     def init(self, conn, logSql, replicaVar=1):
         self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
@@ -61,7 +63,7 @@ class TDTestCase:
 
     def prepare_data(self):
         for db in self.dbnames:
-            tdSql.execute(f"create database {db}")
+            tdSql.execute(f"create database {db} vgroups 1")
             tdSql.execute(f"use {db}")
             tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict))
             for i in range(self.tbnum):
@@ -110,11 +112,9 @@ class TDTestCase:
                         tdLog.exit(f"grant privilege, should get res")
                 elif cnt == 2:
                     if res:
-                        time.sleep(1000)
                         tdLog.exit(f"revoke privilege, should get NULL")
-
-                else:
-                    break
+                    else:
+                        break
 
                 tdLog.debug("test subscribe topic privilege revoked by other user")
                 tdSql.execute(f'revoke subscribe on {self.topic_name} from {self.user_name}')
@@ -122,7 +122,6 @@ class TDTestCase:
 
         finally:
             consumer.close()
-        time.sleep(1000)
 
     def create_user(self):
         tdSql.execute(f'create topic {self.topic_name} as database {self.dbnames[0]}')

From bd2bef2428ea0f1b462261ce6fde0cae558210de Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 26 Jan 2024 15:05:31 +0800
Subject: [PATCH 04/33] fix:add test case

---
 source/client/src/clientTmq.c                 |  2 +-
 source/common/src/tgrant.c                    | 10 ++++-
 source/dnode/mnode/impl/src/mndConsumer.c     |  2 +-
 source/dnode/mnode/impl/src/mndStream.c       |  6 ++-
 source/libs/parser/src/parTranslater.c        |  8 ++--
 .../0-others/subscribe_stream_privilege.py    | 45 +++++++++++++++++--
 6 files changed, 61 insertions(+), 12 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 79611b7eee..8b424a7bf7 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -762,7 +762,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
       }
       taosWUnLockLatch(&tmq->lock);
     }
-
+    tDeatroySMqHbRsp(&rsp);
     taosMemoryFree(pMsg->pData);
     taosMemoryFree(pMsg->pEpSet);
   }
diff --git a/source/common/src/tgrant.c b/source/common/src/tgrant.c
index 74a59fd580..eb0e677b37 100644
--- a/source/common/src/tgrant.c
+++ b/source/common/src/tgrant.c
@@ -18,6 +18,14 @@
 
 #ifndef _GRANT
 
-int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; }
+int32_t grantCheck(EGrantType grant) {
+  if(taosGetTimestampMs() < 1706252996000) {
+    uError("receivee no expired");
+    return 0;
+  } else{
+    uError("receivee expired");
+    return -1;
+  }
+}
 
 #endif
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 2c8a193121..ffa0fbda12 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -101,7 +101,7 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
       goto FAILED;
     }
 
-    if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) {
+    if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIBE) < 0) {
       code = TSDB_CODE_MND_NO_RIGHTS;
       terrno = TSDB_CODE_MND_NO_RIGHTS;
       goto FAILED;
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index bf92a51ed8..79e39df581 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1907,9 +1907,10 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
 
   if (pStream == NULL) {
     if (pauseReq.igNotExists) {
-      mInfo("stream:%s, not exist, if exist is set", pauseReq.name);
+      mInfo("stream:%s, not exist 1, if exist is set", pauseReq.name);
       return 0;
     } else {
+      mInfo("stream:%s, not exist 2, if exist is set,%p,%d,%p", pauseReq.name, pReq->pCont, pReq->contLen, pReq);
       terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
       return -1;
     }
@@ -3066,6 +3067,7 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){
       };
 
       tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+      mInfo("receivee pause stream:%s, %s, %p, because grant expired", pStream->name, reqPause->name, reqPause->name);
     }
 
     sdbRelease(pSdb, pStream);
@@ -3099,7 +3101,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   }
   tDecoderClear(&decoder);
 
-  mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks);
+  mDebug("receivee stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks);
 
   taosThreadMutexLock(&execInfo.lock);
 
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index d246641576..5aaa1a815c 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -3923,7 +3923,7 @@ static int32_t checkStateWindowForStream(STranslateContext* pCxt, SSelectStmt* p
   }
   if (TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType &&
       !hasPartitionByTbname(pSelect->pPartitionByList)) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query1");
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -7467,7 +7467,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt
 
   if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery) || NULL == ((SSelectStmt*)pStmt->pQuery)->pFromTable ||
       QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pStmt->pQuery)->pFromTable)) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query2");
   }
 
 #ifdef TD_ENTERPRISE
@@ -7486,7 +7486,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
   }
   if (TSDB_VIEW_TABLE == tableType) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query3");
   }
 #endif  
   
@@ -7721,7 +7721,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
   if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type ||
       !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList ||
       crossTableWithUdaf(pSelect) || hasJsonTypeProjection(pSelect)) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query4");
   }
   if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) {
     return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py
index 5f40450af4..44778d39d8 100644
--- a/tests/system-test/0-others/subscribe_stream_privilege.py
+++ b/tests/system-test/0-others/subscribe_stream_privilege.py
@@ -23,7 +23,7 @@ from util.sqlset import *
 
 class TDTestCase:
     clientCfgDict = {'debugFlag': 135}
-    updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict}
+    updatecfgDict = {'debugFlag': 143, 'clientCfg':clientCfgDict}
     def init(self, conn, logSql, replicaVar=1):
         self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
@@ -71,6 +71,41 @@ class TDTestCase:
                 for j in self.values_list:
                     tdSql.execute(f'insert into {self.stbname}_{i} values({j})')
 
+    def checkUserPrivileges(self, rowCnt):
+        tdSql.query("show user privileges")
+        tdSql.checkRows(rowCnt)
+
+    def streamTest(self):
+        tdSql.execute("create stream s1 trigger at_once fill_history 1 into so1 as select ts,abs(col2) from stb partition by tbname")
+        time.sleep(2)
+        tdSql.query("select * from so1")
+        tdSql.checkRows(4)
+        tdSql.execute("insert into stb_0(ts,col2) values(now, 332)")
+        time.sleep(2)
+        tdSql.query("select * from so1")
+        tdSql.checkRows(5)
+
+        time.sleep(2)
+        tdSql.query("select * from information_schema.ins_stream_tasks")
+        tdSql.checkData(0, 5, 'ready')
+
+        print(time.time())
+        while 1:
+            t = time.time()
+            if t > 1706252996 :
+                break
+            else:
+                print("time:%d" %(t))
+                time.sleep(1)
+
+
+        tdSql.error("create stream s11 trigger at_once fill_history 1 into so1 as select ts,abs(col2) from stb partition by tbname")
+        tdSql.query("select * from information_schema.ins_stream_tasks")
+        tdSql.checkData(0, 5, 'pause')
+        tdSql.execute("insert into stb_0(ts,col2) values(now, 3232)")
+        tdSql.query("select * from so1")
+        tdSql.checkRows(5)
+
     def consumeTest(self):
         consumer_dict = {
             "group.id": "g1",
@@ -90,8 +125,10 @@ class TDTestCase:
         if not exceptOccured:
             tdLog.exit(f"has no privilege, should except")
 
+        checkUserPrivileges(1)
         tdLog.debug("test subscribe topic privilege granted by other user")
         tdSql.execute(f'grant subscribe on {self.topic_name} to {self.user_name}')
+        checkUserPrivileges(2)
 
         exceptOccured = False
         try:
@@ -118,6 +155,7 @@ class TDTestCase:
 
                 tdLog.debug("test subscribe topic privilege revoked by other user")
                 tdSql.execute(f'revoke subscribe on {self.topic_name} from {self.user_name}')
+                checkUserPrivileges(1)
                 time.sleep(5)
 
         finally:
@@ -130,8 +168,9 @@ class TDTestCase:
     def run(self):
         self.prepare_data()
         self.create_user()
-        self.consumeTest()
-                
+        #self.consumeTest()
+        self.streamTest()
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

From ce2a3e4be2a9113d4daf55bb5ba22089578b5270 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 26 Jan 2024 15:35:29 +0800
Subject: [PATCH 05/33] fix:test case error

---
 source/common/src/tgrant.c                    |  2 +-
 source/dnode/mnode/impl/src/mndStream.c       | 21 +++++++++----------
 .../0-others/subscribe_stream_privilege.py    |  8 +++++--
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/source/common/src/tgrant.c b/source/common/src/tgrant.c
index eb0e677b37..6251131005 100644
--- a/source/common/src/tgrant.c
+++ b/source/common/src/tgrant.c
@@ -19,7 +19,7 @@
 #ifndef _GRANT
 
 int32_t grantCheck(EGrantType grant) {
-  if(taosGetTimestampMs() < 1706252996000) {
+  if(taosGetTimestampMs() < 1706254434000) {
     uError("receivee no expired");
     return 0;
   } else{
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 79e39df581..4b753fb5a6 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -3050,24 +3050,23 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){
     if (pIter == NULL) break;
 
     if(pStream->status != STREAM_STATUS__PAUSE){
-      SMPauseStreamReq *reqPause = rpcMallocCont(sizeof(SMPauseStreamReq));
-      if (reqPause == NULL) {
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        sdbRelease(pSdb, pStream);
-        return -1;
-      }
-      strcpy(reqPause->name, pStream->name);
-      reqPause->igNotExists = 1;
+      SMPauseStreamReq reqPause = {0};
+      strcpy(reqPause.name, pStream->name);
+      reqPause.igNotExists = 1;
+
+      int32_t contLen = tSerializeSMPauseStreamReq(NULL, 0, &reqPause);
+      void *  pHead = rpcMallocCont(contLen);
+      tSerializeSMPauseStreamReq(pHead, contLen, &reqPause);
 
       SRpcMsg rpcMsg = {
           .msgType = TDMT_MND_PAUSE_STREAM,
-          .pCont = reqPause,
-          .contLen = sizeof(SMPauseStreamReq),
+          .pCont = pHead,
+          .contLen = contLen,
           .info = *info,
       };
 
       tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
-      mInfo("receivee pause stream:%s, %s, %p, because grant expired", pStream->name, reqPause->name, reqPause->name);
+      mInfo("receivee pause stream:%s, %s, %p, because grant expired", pStream->name, reqPause.name, reqPause.name);
     }
 
     sdbRelease(pSdb, pStream);
diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py
index 44778d39d8..9656d0e5e8 100644
--- a/tests/system-test/0-others/subscribe_stream_privilege.py
+++ b/tests/system-test/0-others/subscribe_stream_privilege.py
@@ -92,7 +92,7 @@ class TDTestCase:
         print(time.time())
         while 1:
             t = time.time()
-            if t > 1706252996 :
+            if t > 1706254434 :
                 break
             else:
                 print("time:%d" %(t))
@@ -100,12 +100,16 @@ class TDTestCase:
 
 
         tdSql.error("create stream s11 trigger at_once fill_history 1 into so1 as select ts,abs(col2) from stb partition by tbname")
+
+        time.sleep(10)
         tdSql.query("select * from information_schema.ins_stream_tasks")
-        tdSql.checkData(0, 5, 'pause')
+        tdSql.checkData(0, 5, 'paused')
         tdSql.execute("insert into stb_0(ts,col2) values(now, 3232)")
         tdSql.query("select * from so1")
         tdSql.checkRows(5)
 
+        tdSql.error("resume stream s1")
+
     def consumeTest(self):
         consumer_dict = {
             "group.id": "g1",

From 388e89096c301666fbe2018136bccf81b0ec0b67 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 26 Jan 2024 15:51:37 +0800
Subject: [PATCH 06/33] fix:test case error

---
 source/common/src/tgrant.c                             | 10 +---------
 tests/parallel_test/cases.task                         |  1 +
 .../system-test/0-others/subscribe_stream_privilege.py |  4 ++--
 3 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/source/common/src/tgrant.c b/source/common/src/tgrant.c
index 6251131005..f212d71362 100644
--- a/source/common/src/tgrant.c
+++ b/source/common/src/tgrant.c
@@ -18,14 +18,6 @@
 
 #ifndef _GRANT
 
-int32_t grantCheck(EGrantType grant) {
-  if(taosGetTimestampMs() < 1706254434000) {
-    uError("receivee no expired");
-    return 0;
-  } else{
-    uError("receivee expired");
-    return -1;
-  }
-}
+int32_t grantCheck(EGrantType grant) {return TSDB_CODE_SUCCESS;}
 
 #endif
\ No newline at end of file
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 79bec1ec76..97f4e533db 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -292,6 +292,7 @@ fi
 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py
 
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py
diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py
index 9656d0e5e8..d85d87dc3a 100644
--- a/tests/system-test/0-others/subscribe_stream_privilege.py
+++ b/tests/system-test/0-others/subscribe_stream_privilege.py
@@ -172,8 +172,8 @@ class TDTestCase:
     def run(self):
         self.prepare_data()
         self.create_user()
-        #self.consumeTest()
-        self.streamTest()
+        self.consumeTest()
+        # self.streamTest()
 
     def stop(self):
         tdSql.close()

From 7c828d35064ad666f60ed3b7fa36ada705ac8e42 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 26 Jan 2024 16:42:26 +0800
Subject: [PATCH 07/33] feat:[TD-28247]add grant for subscribe and stream

---
 include/common/tmsg.h                  | 1 -
 source/common/src/tmsg.c               | 2 --
 source/libs/parser/src/parTranslater.c | 8 ++++----
 tests/parallel_test/cases.task         | 2 +-
 4 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 05e2738f7c..5cf18fbb66 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3360,7 +3360,6 @@ typedef struct {
   char   name[TSDB_STREAM_FNAME_LEN];
   int8_t igNotExists;
   int8_t igUntreated;
-  int8_t suspend;
 } SMResumeStreamReq;
 
 int32_t tSerializeSMResumeStreamReq(void* buf, int32_t bufLen, const SMResumeStreamReq* pReq);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index a395884538..3f1cfbc87f 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -8933,7 +8933,6 @@ int32_t tSerializeSMResumeStreamReq(void *buf, int32_t bufLen, const SMResumeStr
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igUntreated) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->suspend) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -8948,7 +8947,6 @@ int32_t tDeserializeSMResumeStreamReq(void *buf, int32_t bufLen, SMResumeStreamR
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igUntreated) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->suspend) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 5aaa1a815c..d246641576 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -3923,7 +3923,7 @@ static int32_t checkStateWindowForStream(STranslateContext* pCxt, SSelectStmt* p
   }
   if (TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType &&
       !hasPartitionByTbname(pSelect->pPartitionByList)) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query1");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -7467,7 +7467,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt
 
   if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery) || NULL == ((SSelectStmt*)pStmt->pQuery)->pFromTable ||
       QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pStmt->pQuery)->pFromTable)) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query2");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
   }
 
 #ifdef TD_ENTERPRISE
@@ -7486,7 +7486,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
   }
   if (TSDB_VIEW_TABLE == tableType) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query3");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
   }
 #endif  
   
@@ -7721,7 +7721,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
   if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type ||
       !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList ||
       crossTableWithUdaf(pSelect) || hasJsonTypeProjection(pSelect)) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query4");
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
   }
   if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) {
     return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 97f4e533db..0dfbdf3038 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -292,7 +292,7 @@ fi
 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py
+,,n,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py
 
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py

From f86a8248d2b1589179232b452c33b78ef080c48b Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Mon, 29 Jan 2024 10:19:01 +0800
Subject: [PATCH 08/33] fix: remove stmt assert

---
 source/client/src/clientStmt.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c
index 8ac9550aca..36a3e50aef 100644
--- a/source/client/src/clientStmt.c
+++ b/source/client/src/clientStmt.c
@@ -406,10 +406,6 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
 
   if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) {
     if (pStmt->bInfo.inExecCache) {
-      if (ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1)) {
-        tscError("stmtGetFromCache error");
-        return TSDB_CODE_TSC_STMT_CACHE_ERROR;
-      }
       pStmt->bInfo.needParse = false;
       tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName);
       return TSDB_CODE_SUCCESS;

From 5f1b48a8daf93659a5da2a5f5cbe699c3afdc437 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 29 Jan 2024 10:50:10 +0800
Subject: [PATCH 09/33] fix:[TS-4479] support long varbinary in format of
 \x3423

---
 source/libs/parser/src/parInsertSql.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c
index 1994ddb437..ce2d51c911 100644
--- a/source/libs/parser/src/parInsertSql.c
+++ b/source/libs/parser/src/parInsertSql.c
@@ -458,11 +458,13 @@ static int32_t parseVarbinary(SToken* pToken, uint8_t **pData, uint32_t *nData,
     *pData = data;
     *nData = size;
   }else{
-    if (pToken->n + VARSTR_HEADER_SIZE > bytes) {
+    *pData = taosMemoryCalloc(1, pToken->n);
+    int32_t len = trimString(pToken->z, pToken->n, *pData, pToken->n);
+    *nData = len;
+
+    if (*nData + VARSTR_HEADER_SIZE > bytes) {
       return TSDB_CODE_PAR_VALUE_TOO_LONG;
     }
-    *pData = taosStrdup(pToken->z);
-    *nData = pToken->n;
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -753,7 +755,7 @@ static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* p
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) {
+static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type) {
   if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER &&
        pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL &&
        pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT &&
@@ -763,7 +765,7 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs
   }
 
   // Remove quotation marks
-  if (TK_NK_STRING == pToken->type) {
+  if (TK_NK_STRING == pToken->type && type != TSDB_DATA_TYPE_VARBINARY) {
     if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
       return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
     }
@@ -935,7 +937,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt
 
     SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]];
     isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON;
-    code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg);
+    code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg, pTagSchema->type);
     if (TK_NK_VARIABLE == token.type) {
       code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z);
     }
@@ -1631,7 +1633,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
 
 static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema,
                                int16_t timePrec, SColVal* pVal) {
-  int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
+  int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, pSchema->type);
   if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) {
     if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
       return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z);
@@ -1691,7 +1693,7 @@ typedef union SRowsDataContext{
 
 static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) {
   *pFoundCtbName = false;
-  int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
+  int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, TSDB_DATA_TYPE_BINARY);
   if (TK_NK_VARIABLE == pToken->type) {
     code = buildInvalidOperationMsg(&pCxt->msg, "not expected tbname");
   }
@@ -1731,7 +1733,7 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif
     for (int32_t i = 0; code == TSDB_CODE_SUCCESS && i < numOfTagTokens; ++i) {
       SToken*  pTagToken = (SToken*)(tagTokens + i);
       SSchema* pTagSchema = tagSchemas[i];
-      code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg);
+      code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg, pTagSchema->type);
       if (TK_NK_VARIABLE == pTagToken->type) {
         code = buildInvalidOperationMsg(&pCxt->msg, "not expected tag");
       }
@@ -1790,7 +1792,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
         tagSchemas[(*pNumOfTagTokens)] = (SSchema*)pTagSchema;
         ++(*pNumOfTagTokens);
       } else {
-        code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
+        code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, pTagSchema->type);
         if (TK_NK_VARIABLE == pToken->type) {
           code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value");
         }

From 9087a0d9d049c2808d353b92e1d9499c2358ce90 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 29 Jan 2024 11:03:50 +0800
Subject: [PATCH 10/33] feat(stream): drop orphan tasks.

---
 include/common/tmsg.h                       |   2 +-
 source/dnode/mnode/impl/inc/mndStream.h     |  16 +-
 source/dnode/mnode/impl/src/mndSma.c        |   4 +-
 source/dnode/mnode/impl/src/mndStream.c     |  48 +-----
 source/dnode/mnode/impl/src/mndStreamHb.c   |  84 ++++++++--
 source/dnode/mnode/impl/src/mndStreamUtil.c | 171 ++++++++++++++++++--
 6 files changed, 237 insertions(+), 88 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index c314d82036..c4da3194e0 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3323,7 +3323,7 @@ typedef struct {
   SMsgHead head;
   int64_t  streamId;
   int32_t  taskId;
-} SVPauseStreamTaskReq, SVResetStreamTaskReq, SVDropHTaskReq;
+} SVPauseStreamTaskReq, SVResetStreamTaskReq;
 
 typedef struct {
   int8_t reserved;
diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index 92035101f6..d884227249 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -69,12 +69,6 @@ typedef struct SNodeEntry {
   int64_t hbTimestamp;   // second
 } SNodeEntry;
 
-typedef struct SFailedCheckpointInfo {
-  int64_t streamUid;
-  int64_t checkpointId;
-  int32_t transId;
-} SFailedCheckpointInfo;
-
 #define MND_STREAM_CREATE_NAME      "stream-create"
 #define MND_STREAM_CHECKPOINT_NAME  "stream-checkpoint"
 #define MND_STREAM_PAUSE_NAME       "stream-pause"
@@ -97,9 +91,14 @@ int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId)
 bool    mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamUid, const char *pTransName, bool lock);
 int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamUid);
 
+typedef struct SOrphanTask {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t nodeId;
+} SOrphanTask;
+
 // for sma
 // TODO refactor
-int32_t     mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t     mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams);
 int32_t     mndGetNumOfStreamTasks(const SStreamObj *pStream);
 SArray     *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
@@ -119,7 +118,8 @@ void        saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode)
 int32_t     initStreamNodeList(SMnode *pMnode);
 int32_t     mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj* pStream, int8_t igUntreated);
 int32_t     mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
-
+int32_t     mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+int32_t     mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c
index a89136e7d3..e6027a0332 100644
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@@ -865,7 +865,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p
     sdbRelease(pMnode->pSdb, pStream);
     goto _OVER;
   } else {
-    if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
+    if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
       mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
       sdbRelease(pMnode->pSdb, pStream);
       goto _OVER;
@@ -917,7 +917,7 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
 
       SStreamObj *pStream = mndAcquireStream(pMnode, streamName);
       if (pStream != NULL && pStream->smaId == pSma->uid) {
-        if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
+        if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
           mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
           mndReleaseStream(pMnode, pStream);
           goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 7b348172f2..46c7a06079 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -608,50 +608,6 @@ _OVER:
   return -1;
 }
 
-static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
-  SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
-  if (pReq == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pReq->head.vgId = htonl(pTask->info.nodeId);
-  pReq->taskId = pTask->id.taskId;
-  pReq->streamId = pTask->id.streamId;
-
-  SEpSet  epset = {0};
-  bool    hasEpset = false;
-  int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
-  if (code != TSDB_CODE_SUCCESS || !hasEpset) {  // no valid epset, return directly without redoAction
-    terrno = code;
-    return -1;
-  }
-
-  // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
-  code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
-  if (code != 0) {
-    taosMemoryFree(pReq);
-    return -1;
-  }
-
-  return 0;
-}
-
-int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
-  int32_t lv = taosArrayGetSize(pStream->tasks);
-  for (int32_t i = 0; i < lv; i++) {
-    SArray *pTasks = taosArrayGetP(pStream->tasks, i);
-    int32_t sz = taosArrayGetSize(pTasks);
-    for (int32_t j = 0; j < sz; j++) {
-      SStreamTask *pTask = taosArrayGetP(pTasks, j);
-      if (mndPersistTaskDropReq(pMnode, pTrans, pTask) < 0) {
-        return -1;
-      }
-    }
-  }
-  return 0;
-}
-
 static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) {  // check for number of existed tasks
   int32_t     numOfStream = 0;
   SStreamObj *pStream = NULL;
@@ -1200,7 +1156,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
   int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid);
 
   // drop all tasks
-  if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
+  if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
     mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr());
     sdbRelease(pMnode->pSdb, pStream);
     mndTransDrop(pTrans);
@@ -1264,7 +1220,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
         return -1;
       } else {
 #if 0
-        if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
+        if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
           mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
           sdbRelease(pMnode->pSdb, pStream);
           sdbCancelFetch(pSdb, pIter);
diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c
index e4599edbd4..5a6faadebb 100644
--- a/source/dnode/mnode/impl/src/mndStreamHb.c
+++ b/source/dnode/mnode/impl/src/mndStreamHb.c
@@ -16,6 +16,12 @@
 #include "mndStream.h"
 #include "mndTrans.h"
 
+typedef struct SFailedCheckpointInfo {
+  int64_t streamUid;
+  int64_t checkpointId;
+  int32_t transId;
+} SFailedCheckpointInfo;
+
 static void doExtractTasksFromStream(SMnode *pMnode) {
   SSdb       *pSdb = pMnode->pSdb;
   SStreamObj *pStream = NULL;
@@ -177,10 +183,51 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
+  SOrphanTask* pTask = taosArrayGet(pList, 0);
+
+  // check if it is conflict with other trans in both sourceDb and targetDb.
+  bool conflict = mndStreamTransConflictCheck(pMnode, pTask->streamId, MND_STREAM_DROP_NAME, false);
+  if (conflict) {
+    return -1;
+  }
+
+  SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
+  STrans* pTrans = doCreateTrans(pMnode, &dummyObj, NULL, MND_STREAM_DROP_NAME, "drop stream");
+  if (pTrans == NULL) {
+    mError("failed to create trans to drop orphan tasks since %s", terrstr());
+    return -1;
+  }
+
+  int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId);
+
+  // drop all tasks
+  if (mndStreamSetDropActionFromList(pMnode, pTrans, pList) < 0) {
+    mError("failed to create trans to drop orphan tasks since %s", terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  // drop stream
+  if (mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED) < 0) {
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  return 0;
+}
+
 int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   SMnode      *pMnode = pReq->info.node;
   SStreamHbMsg req = {0};
-  SArray      *pList = taosArrayInit(4, sizeof(SFailedCheckpointInfo));
+  SArray      *pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo));
+  SArray      *pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask));
 
   SDecoder decoder = {0};
   tDecoderInit(&decoder, pReq->pCont, pReq->contLen);
@@ -198,8 +245,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   taosThreadMutexLock(&execInfo.lock);
 
   // extract stream task list
-  int32_t numOfExisted = taosHashGetSize(execInfo.pTaskMap);
-  if (numOfExisted == 0) {
+  if (taosHashGetSize(execInfo.pTaskMap) == 0) {
     doExtractTasksFromStream(pMnode);
   }
 
@@ -218,6 +264,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
     STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id));
     if (pTaskEntry == NULL) {
       mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId);
+
+      SOrphanTask oTask = {.streamId = p->id.streamId, .taskId = p->id.taskId, .nodeId = p->nodeId};
+      taosArrayPush(pOrphanTasks, &oTask);
       continue;
     }
 
@@ -240,15 +289,13 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
       }
 
       streamTaskStatusCopy(pTaskEntry, p);
-      if (p->checkpointId != 0) {
-        if (p->checkpointFailed) {
-          mError("stream task:0x%" PRIx64 " checkpointId:%" PRIx64 " transId:%d failed, kill it", p->id.taskId,
-              p->checkpointId, p->chkpointTransId);
+      if ((p->checkpointId != 0) && p->checkpointFailed) {
+        mError("stream task:0x%" PRIx64 " checkpointId:%" PRIx64 " transId:%d failed, kill it", p->id.taskId,
+               p->checkpointId, p->chkpointTransId);
 
-          SFailedCheckpointInfo info = {
-              .transId = p->chkpointTransId, .checkpointId = p->checkpointId, .streamUid = p->id.streamId};
-          addIntoCheckpointList(pList, &info);
-        }
+        SFailedCheckpointInfo info = {
+            .transId = p->chkpointTransId, .checkpointId = p->checkpointId, .streamUid = p->id.streamId};
+        addIntoCheckpointList(pFailedTasks, &info);
       }
     }
 
@@ -266,15 +313,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
 
   // current checkpoint is failed, rollback from the checkpoint trans
   // kill the checkpoint trans and then set all tasks status to be normal
-  if (taosArrayGetSize(pList) > 0) {
+  if (taosArrayGetSize(pFailedTasks) > 0) {
     bool    allReady = true;
     SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady);
     taosArrayDestroy(p);
 
     if (allReady || snodeChanged) {
       // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
-      for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
-        SFailedCheckpointInfo *pInfo = taosArrayGet(pList, i);
+      for(int32_t i = 0; i < taosArrayGetSize(pFailedTasks); ++i) {
+        SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedTasks, i);
         mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status",
             pInfo->checkpointId, pInfo->transId);
 
@@ -285,9 +332,16 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
     }
   }
 
+  // handle the orphan tasks that are invalid but not removed in some vnodes or snode due to some unknown errors.
+  if (taosArrayGetSize(pOrphanTasks) > 0) {
+    mndDropOrphanTasks(pMnode, pOrphanTasks);
+  }
+
   taosThreadMutexUnlock(&execInfo.lock);
   streamMetaClearHbMsg(&req);
 
-  taosArrayDestroy(pList);
+  taosArrayDestroy(pFailedTasks);
+  taosArrayDestroy(pOrphanTasks);
+
   return TSDB_CODE_SUCCESS;
 }
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index 2ee73528e0..5d6e34856b 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -18,6 +18,66 @@
 #include "tmisce.h"
 #include "mndVgroup.h"
 
+typedef struct SStreamTaskIter {
+  SStreamObj  *pStream;
+  int32_t      level;
+  int32_t      ordinalIndex;
+  int32_t      totalLevel;
+  SStreamTask *pTask;
+} SStreamTaskIter;
+
+SStreamTaskIter* createTaskIter(SStreamObj* pStream) {
+  SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
+  if (pIter == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pIter->level = -1;
+  pIter->ordinalIndex = 0;
+  pIter->pStream = pStream;
+  pIter->totalLevel = taosArrayGetSize(pStream->tasks);
+  pIter->pTask = NULL;
+
+  return pIter;
+}
+
+bool taskIterNextTask(SStreamTaskIter* pIter) {
+  if (pIter->level >= pIter->totalLevel) {
+    pIter->pTask = NULL;
+    return false;
+  }
+
+  if (pIter->level == -1) {
+    pIter->level += 1;
+  }
+
+  while(pIter->level < pIter->totalLevel) {
+    SArray *pList = taosArrayGetP(pIter->pStream->tasks, pIter->level);
+    if (pIter->ordinalIndex >= taosArrayGetSize(pList)) {
+      pIter->level += 1;
+      pIter->ordinalIndex = 0;
+      pIter->pTask = NULL;
+      continue;
+    }
+
+    pIter->pTask = taosArrayGetP(pList, pIter->ordinalIndex);
+    pIter->ordinalIndex += 1;
+    return true;
+  }
+
+  pIter->pTask = NULL;
+  return false;
+}
+
+SStreamTask* taskIterGetCurrent(SStreamTaskIter* pIter) {
+  return pIter->pTask;
+}
+
+void destroyTaskIter(SStreamTaskIter* pIter) {
+  taosMemoryFree(pIter);
+}
+
 SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
   SSdb   *pSdb = pMnode->pSdb;
   void   *pIter = NULL;
@@ -251,24 +311,103 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
 }
 
 int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
-  SArray *tasks = pStream->tasks;
+  SStreamTaskIter *pIter = createTaskIter(pStream);
 
-  int32_t size = taosArrayGetSize(tasks);
-  for (int32_t i = 0; i < size; i++) {
-    SArray *pTasks = taosArrayGetP(tasks, i);
-    int32_t sz = taosArrayGetSize(pTasks);
-    for (int32_t j = 0; j < sz; j++) {
-      SStreamTask *pTask = taosArrayGetP(pTasks, j);
-
-      if (doSetPauseAction(pMnode, pTrans, pTask) < 0) {
-        return -1;
-      }
-
-      if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) {
-        atomic_store_8(&pTask->status.statusBackup, pTask->status.taskStatus);
-        atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE);
-      }
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
+    if (doSetPauseAction(pMnode, pTrans, pTask) < 0) {
+      destroyTaskIter(pIter);
+      return -1;
     }
+
+    if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) {
+      atomic_store_8(&pTask->status.statusBackup, pTask->status.taskStatus);
+      atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE);
+    }
+  }
+
+  destroyTaskIter(pIter);
+  return 0;
+}
+
+static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
+  SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
+  if (pReq == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pReq->head.vgId = htonl(pTask->info.nodeId);
+  pReq->taskId = pTask->id.taskId;
+  pReq->streamId = pTask->id.streamId;
+
+  SEpSet  epset = {0};
+  bool    hasEpset = false;
+  int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
+  if (code != TSDB_CODE_SUCCESS || !hasEpset) {  // no valid epset, return directly without redoAction
+    terrno = code;
+    return -1;
+  }
+
+  // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
+  code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
+  if (code != 0) {
+    taosMemoryFree(pReq);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
+  SStreamTaskIter *pIter = createTaskIter(pStream);
+
+  while(taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
+    if (doSetDropAction(pMnode, pTrans, pTask) < 0) {
+      destroyTaskIter(pIter);
+      return -1;
+    }
+  }
+  destroyTaskIter(pIter);
+  return 0;
+}
+
+static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask* pTask) {
+  SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
+  if (pReq == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pReq->head.vgId = htonl(pTask->nodeId);
+  pReq->taskId = pTask->taskId;
+  pReq->streamId = pTask->streamId;
+
+  SEpSet  epset = {0};
+  bool    hasEpset = false;
+  int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId);
+  if (code != TSDB_CODE_SUCCESS || (!hasEpset)) {  // no valid epset, return directly without redoAction
+    terrno = code;
+    taosMemoryFree(pReq);
+    return -1;
+  }
+
+  // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
+  code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0);
+  if (code != 0) {
+    taosMemoryFree(pReq);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) {
+  for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+    SOrphanTask* pTask = taosArrayGet(pList, i);
+    mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId);
+    doSetDropActionFromId(pMnode, pTrans, pTask);
   }
   return 0;
 }
\ No newline at end of file

From b714c704348210ae84fbb18d34ae2033cebeb0e8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 29 Jan 2024 13:46:33 +0800
Subject: [PATCH 11/33] refactor: do some internal refactor.

---
 source/dnode/mnode/impl/inc/mndStream.h      |   8 +
 source/dnode/mnode/impl/src/mndStream.c      | 177 +++++++++---------
 source/dnode/mnode/impl/src/mndStreamHb.c    |  53 +-----
 source/dnode/mnode/impl/src/mndStreamTrans.c |  82 ---------
 source/dnode/mnode/impl/src/mndStreamUtil.c  | 180 ++++++++++++++++---
 5 files changed, 257 insertions(+), 243 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index d884227249..2778d0481e 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -120,6 +120,14 @@ int32_t     mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj*
 int32_t     mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t     mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t     mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
+int32_t     mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+
+typedef struct SStreamTaskIter SStreamTaskIter;
+
+SStreamTaskIter *createTaskIter(SStreamObj *pStream);
+bool             taskIterNextTask(SStreamTaskIter *pIter);
+SStreamTask     *taskIterGetCurrent(SStreamTaskIter *pIter);
+void             destroyTaskIter(SStreamTaskIter *pIter);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 46c7a06079..c78b80d79a 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -476,22 +476,20 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
 }
 
 int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) {
-  int32_t level = taosArrayGetSize(pStream->tasks);
-  for (int32_t i = 0; i < level; i++) {
-    SArray *pLevel = taosArrayGetP(pStream->tasks, i);
-
-    int32_t numOfTasks = taosArrayGetSize(pLevel);
-    for (int32_t j = 0; j < numOfTasks; j++) {
-      SStreamTask *pTask = taosArrayGetP(pLevel, j);
-      if (mndPersistTaskDeployReq(pTrans, pTask) < 0) {
-        return -1;
-      }
+  SStreamTaskIter *pIter = createTaskIter(pStream);
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
+    if (mndPersistTaskDeployReq(pTrans, pTask) < 0) {
+      destroyTaskIter(pIter);
+      return -1;
     }
   }
 
+  destroyTaskIter(pIter);
+
   // persistent stream task for already stored ts data
   if (pStream->conf.fillHistory) {
-    level = taosArrayGetSize(pStream->pHTasksList);
+    int32_t level = taosArrayGetSize(pStream->pHTasksList);
 
     for (int32_t i = 0; i < level; i++) {
       SArray *pLevel = taosArrayGetP(pStream->pHTasksList, i);
@@ -856,6 +854,32 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
   return 0;
 }
 
+static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, int64_t checkpointId,
+                                     int8_t mndTrigger) {
+  void   *buf;
+  int32_t tlen;
+  if (mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId,
+                                        pTask->id.taskId, pTrans->id, mndTrigger) < 0) {
+    taosMemoryFree(buf);
+    return -1;
+  }
+
+  SEpSet  epset = {0};
+  bool    hasEpset = false;
+  int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
+  if (code != TSDB_CODE_SUCCESS || !hasEpset) {
+    taosMemoryFree(buf);
+    return -1;
+  }
+
+  code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
+  if (code != 0) {
+    taosMemoryFree(buf);
+  }
+
+  return code;
+}
+
 static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId,
                                                int8_t mndTrigger, bool lock) {
   int32_t code = -1;
@@ -865,6 +889,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
     return -1;
   }
 
+
   bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock);
   if (conflict) {
     mndAddtoCheckpointWaitingList(pStream, checkpointId);
@@ -887,8 +912,8 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
   pStream->currentTick = 1;
 
   // 1. redo action: broadcast checkpoint source msg for all source vg
-  int32_t totLevel = taosArrayGetSize(pStream->tasks);
-  for (int32_t i = 0; i < totLevel; i++) {
+  int32_t totalLevel = taosArrayGetSize(pStream->tasks);
+  for (int32_t i = 0; i < totalLevel; i++) {
     SArray      *pLevel = taosArrayGetP(pStream->tasks, i);
     SStreamTask *p = taosArrayGetP(pLevel, 0);
 
@@ -896,28 +921,9 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
       int32_t sz = taosArrayGetSize(pLevel);
       for (int32_t j = 0; j < sz; j++) {
         SStreamTask *pTask = taosArrayGetP(pLevel, j);
+        code = doSetCheckpointAction(pMnode, pTrans, pTask, checkpointId, mndTrigger);
 
-        void   *buf;
-        int32_t tlen;
-        if (mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId,
-                                              pTask->id.taskId, pTrans->id, mndTrigger) < 0) {
-          taosWUnLockLatch(&pStream->lock);
-          goto _ERR;
-        }
-
-        SEpSet epset = {0};
-        bool   hasEpset = false;
-        code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
-        if (code != TSDB_CODE_SUCCESS || !hasEpset) {
-          taosMemoryFree(buf);
-          taosWUnLockLatch(&pStream->lock);
-          goto _ERR;
-        }
-
-        code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset,
-                              TSDB_CODE_SYN_PROPOSE_NOT_READY);
-        if (code != 0) {
-          taosMemoryFree(buf);
+        if (code != TSDB_CODE_SUCCESS) {
           taosWUnLockLatch(&pStream->lock);
           goto _ERR;
         }
@@ -1500,21 +1506,19 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
     }
 
     // add row for each task
-    for (int32_t i = 0; i < taosArrayGetSize(pStream->tasks); i++) {
-      SArray *pLevel = taosArrayGetP(pStream->tasks, i);
+    SStreamTaskIter *pIter = createTaskIter(pStream);
+    while (taskIterNextTask(pIter)) {
+      SStreamTask *pTask = taskIterGetCurrent(pIter);
 
-      int32_t numOfLevels = taosArrayGetSize(pLevel);
-      for (int32_t j = 0; j < numOfLevels; j++) {
-        SStreamTask *pTask = taosArrayGetP(pLevel, j);
-        int32_t      code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows);
-        if (code == TSDB_CODE_SUCCESS) {
-          numOfRows++;
-        }
+      int32_t code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows);
+      if (code == TSDB_CODE_SUCCESS) {
+        numOfRows++;
       }
     }
 
-    // unlock
+    destroyTaskIter(pIter);
     taosRUnLockLatch(&pStream->lock);
+
     sdbRelease(pSdb, pStream);
   }
 
@@ -1854,22 +1858,19 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
     }
 
     taosWLockLatch(&pStream->lock);
-    int32_t numOfLevels = taosArrayGetSize(pStream->tasks);
 
-    for (int32_t j = 0; j < numOfLevels; ++j) {
-      SArray *pLevel = taosArrayGetP(pStream->tasks, j);
+    SStreamTaskIter *pTaskIter = createTaskIter(pStream);
+    while (taskIterNextTask(pTaskIter)) {
+      SStreamTask *pTask = taskIterGetCurrent(pTaskIter);
 
-      int32_t numOfTasks = taosArrayGetSize(pLevel);
-      for (int32_t k = 0; k < numOfTasks; ++k) {
-        SStreamTask *pTask = taosArrayGetP(pLevel, k);
-
-        SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId};
-        epsetAssign(&entry.epset, &pTask->info.epSet);
-        taosHashPut(pHash, &entry.nodeId, sizeof(entry.nodeId), &entry, sizeof(entry));
-      }
+      SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId};
+      epsetAssign(&entry.epset, &pTask->info.epSet);
+      taosHashPut(pHash, &entry.nodeId, sizeof(entry.nodeId), &entry, sizeof(entry));
     }
 
+    destroyTaskIter(pTaskIter);
     taosWUnLockLatch(&pStream->lock);
+
     sdbRelease(pSdb, pStream);
   }
 
@@ -2055,58 +2056,50 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
 }
 
 void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
-  int32_t level = taosArrayGetSize(pStream->tasks);
+  SStreamTaskIter *pIter = createTaskIter(pStream);
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
 
-  for (int32_t i = 0; i < level; i++) {
-    SArray *pLevel = taosArrayGetP(pStream->tasks, i);
+    STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
+    void   *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
+    if (p == NULL) {
+      STaskStatusEntry entry = {0};
+      streamTaskStatusInit(&entry, pTask);
 
-    int32_t numOfTasks = taosArrayGetSize(pLevel);
-    for (int32_t j = 0; j < numOfTasks; j++) {
-      SStreamTask *pTask = taosArrayGetP(pLevel, j);
-
-      STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-      void   *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
-      if (p == NULL) {
-        STaskStatusEntry entry = {0};
-        streamTaskStatusInit(&entry, pTask);
-
-        taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry));
-        taosArrayPush(pExecNode->pTaskList, &id);
-        mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId,
-              (int32_t)taosArrayGetSize(pExecNode->pTaskList));
-      }
+      taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry));
+      taosArrayPush(pExecNode->pTaskList, &id);
+      mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId,
+            (int32_t)taosArrayGetSize(pExecNode->pTaskList));
     }
   }
+
+  destroyTaskIter(pIter);
 }
 
 void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
-  int32_t level = taosArrayGetSize(pStream->tasks);
-  for (int32_t i = 0; i < level; i++) {
-    SArray *pLevel = taosArrayGetP(pStream->tasks, i);
+  SStreamTaskIter *pIter = createTaskIter(pStream);
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
 
-    int32_t numOfTasks = taosArrayGetSize(pLevel);
-    for (int32_t j = 0; j < numOfTasks; j++) {
-      SStreamTask *pTask = taosArrayGetP(pLevel, j);
+    STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
+    void   *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
+    if (p != NULL) {
+      taosHashRemove(pExecNode->pTaskMap, &id, sizeof(id));
 
-      STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-      void   *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
-      if (p != NULL) {
-        taosHashRemove(pExecNode->pTaskMap, &id, sizeof(id));
+      for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
+        STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
+        if (pId->taskId == id.taskId && pId->streamId == id.streamId) {
+          taosArrayRemove(pExecNode->pTaskList, k);
 
-        for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
-          STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
-          if (pId->taskId == id.taskId && pId->streamId == id.streamId) {
-            taosArrayRemove(pExecNode->pTaskList, k);
-
-            int32_t num = taosArrayGetSize(pExecNode->pTaskList);
-            mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)id.taskId, num);
-            break;
-          }
+          int32_t num = taosArrayGetSize(pExecNode->pTaskList);
+          mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)id.taskId, num);
+          break;
         }
       }
     }
   }
 
+  destroyTaskIter(pIter);
   ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
 }
 
diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c
index 5a6faadebb..19c4f22280 100644
--- a/source/dnode/mnode/impl/src/mndStreamHb.c
+++ b/source/dnode/mnode/impl/src/mndStreamHb.c
@@ -65,61 +65,24 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI
   taosArrayPush(pList, pInfo);
 }
 
-static int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
+int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
   STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint");
   if (pTrans == NULL) {
     return terrno;
   }
 
   /*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
-
-  taosWLockLatch(&pStream->lock);
-  int32_t numOfLevels = taosArrayGetSize(pStream->tasks);
-
-  for (int32_t j = 0; j < numOfLevels; ++j) {
-    SArray *pLevel = taosArrayGetP(pStream->tasks, j);
-
-    int32_t numOfTasks = taosArrayGetSize(pLevel);
-    for (int32_t k = 0; k < numOfTasks; ++k) {
-      SStreamTask *pTask = taosArrayGetP(pLevel, k);
-
-      // todo extract method, with pause stream task
-      SVResetStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq));
-      if (pReq == NULL) {
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq),
-               tstrerror(TSDB_CODE_OUT_OF_MEMORY));
-        taosWUnLockLatch(&pStream->lock);
-        return terrno;
-      }
-
-      pReq->head.vgId = htonl(pTask->info.nodeId);
-      pReq->taskId = pTask->id.taskId;
-      pReq->streamId = pTask->id.streamId;
-
-      SEpSet  epset = {0};
-      bool    hasEpset = false;
-      int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
-      if (code != TSDB_CODE_SUCCESS || !hasEpset) {
-        taosMemoryFree(pReq);
-        continue;
-      }
-
-      code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0);
-      if (code != 0) {
-        taosMemoryFree(pReq);
-        taosWUnLockLatch(&pStream->lock);
-        mndTransDrop(pTrans);
-        return terrno;
-      }
-    }
+  int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
+  if (code != 0) {
+    sdbRelease(pMnode->pSdb, pStream);
+    mndTransDrop(pTrans);
+    return code;
   }
 
-  taosWUnLockLatch(&pStream->lock);
-
-  int32_t code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
+  code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
   if (code != TSDB_CODE_SUCCESS) {
     sdbRelease(pMnode->pSdb, pStream);
+    mndTransDrop(pTrans);
     return -1;
   }
 
diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c
index 0a7397827e..f8497e14f7 100644
--- a/source/dnode/mnode/impl/src/mndStreamTrans.c
+++ b/source/dnode/mnode/impl/src/mndStreamTrans.c
@@ -301,86 +301,4 @@ void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) {
   mDebug("complete clear checkpoints in Dbs");
 }
 
-static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
-                              int32_t transId) {
-  pMsg->streamId = pId->streamId;
-  pMsg->taskId = pId->taskId;
-  pMsg->transId = transId;
-  pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
-  taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
-}
 
-static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
-                                          SStreamTaskId *pId, int32_t transId) {
-  SStreamTaskNodeUpdateMsg req = {0};
-  initNodeUpdateMsg(&req, pInfo, pId, transId);
-
-  int32_t code = 0;
-  int32_t blen;
-
-  tEncodeSize(tEncodeStreamTaskUpdateMsg, &req, blen, code);
-  if (code < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosArrayDestroy(req.pNodeList);
-    return -1;
-  }
-
-  int32_t tlen = sizeof(SMsgHead) + blen;
-
-  void *buf = taosMemoryMalloc(tlen);
-  if (buf == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosArrayDestroy(req.pNodeList);
-    return -1;
-  }
-
-  void    *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  SEncoder encoder;
-  tEncoderInit(&encoder, abuf, tlen);
-  tEncodeStreamTaskUpdateMsg(&encoder, &req);
-
-  SMsgHead *pMsgHead = (SMsgHead *)buf;
-  pMsgHead->contLen = htonl(tlen);
-  pMsgHead->vgId = htonl(nodeId);
-
-  tEncoderClear(&encoder);
-
-  *pBuf = buf;
-  *pLen = tlen;
-
-  taosArrayDestroy(req.pNodeList);
-  return TSDB_CODE_SUCCESS;
-}
-
-// todo extract method: traverse stream tasks
-// build trans to update the epset
-int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
-  mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
-
-  taosWLockLatch(&pStream->lock);
-  int32_t numOfLevels = taosArrayGetSize(pStream->tasks);
-
-  for (int32_t j = 0; j < numOfLevels; ++j) {
-    SArray *pLevel = taosArrayGetP(pStream->tasks, j);
-
-    int32_t numOfTasks = taosArrayGetSize(pLevel);
-    for (int32_t k = 0; k < numOfTasks; ++k) {
-      SStreamTask *pTask = taosArrayGetP(pLevel, k);
-
-      void   *pBuf = NULL;
-      int32_t len = 0;
-      streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
-      doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
-
-      int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
-      if (code != TSDB_CODE_SUCCESS) {
-        taosMemoryFree(pBuf);
-        taosWUnLockLatch(&pStream->lock);
-        return -1;
-      }
-    }
-  }
-
-  taosWUnLockLatch(&pStream->lock);
-  return 0;
-}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index 5d6e34856b..c28ba43f65 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -18,13 +18,13 @@
 #include "tmisce.h"
 #include "mndVgroup.h"
 
-typedef struct SStreamTaskIter {
+struct SStreamTaskIter {
   SStreamObj  *pStream;
   int32_t      level;
   int32_t      ordinalIndex;
   int32_t      totalLevel;
   SStreamTask *pTask;
-} SStreamTaskIter;
+};
 
 SStreamTaskIter* createTaskIter(SStreamObj* pStream) {
   SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
@@ -235,18 +235,16 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
 }
 
 SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) {
-  for (int32_t i = 0; i < taosArrayGetSize(pStream->tasks); i++) {
-    SArray *pLevel = taosArrayGetP(pStream->tasks, i);
-
-    int32_t numOfLevels = taosArrayGetSize(pLevel);
-    for (int32_t j = 0; j < numOfLevels; j++) {
-      SStreamTask *pTask = taosArrayGetP(pLevel, j);
-      if (pTask->id.taskId == pId->taskId) {
-        return pTask;
-      }
+  SStreamTaskIter *pIter = createTaskIter(pStream);
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
+    if (pTask->id.taskId == pId->taskId) {
+      destroyTaskIter(pIter);
+      return pTask;
     }
   }
 
+  destroyTaskIter(pIter);
   return NULL;
 }
 
@@ -261,21 +259,20 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
 }
 
 int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) {
-  int32_t size = taosArrayGetSize(pStream->tasks);
-  for (int32_t i = 0; i < size; i++) {
-    SArray *pTasks = taosArrayGetP(pStream->tasks, i);
-    int32_t sz = taosArrayGetSize(pTasks);
-    for (int32_t j = 0; j < sz; j++) {
-      SStreamTask *pTask = taosArrayGetP(pTasks, j);
-      if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) {
-        return -1;
-      }
+  SStreamTaskIter *pIter = createTaskIter(pStream);
 
-      if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) {
-        atomic_store_8(&pTask->status.taskStatus, pTask->status.statusBackup);
-      }
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
+    if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) {
+      destroyTaskIter(pIter);
+      return -1;
+    }
+
+    if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) {
+      atomic_store_8(&pTask->status.taskStatus, pTask->status.statusBackup);
     }
   }
+  destroyTaskIter(pIter);
   return 0;
 }
 
@@ -410,4 +407,139 @@ int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* p
     doSetDropActionFromId(pMnode, pTrans, pTask);
   }
   return 0;
-}
\ No newline at end of file
+}
+
+static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
+                              int32_t transId) {
+  pMsg->streamId = pId->streamId;
+  pMsg->taskId = pId->taskId;
+  pMsg->transId = transId;
+  pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
+  taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
+}
+
+static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
+                                          SStreamTaskId *pId, int32_t transId) {
+  SStreamTaskNodeUpdateMsg req = {0};
+  initNodeUpdateMsg(&req, pInfo, pId, transId);
+
+  int32_t code = 0;
+  int32_t blen;
+
+  tEncodeSize(tEncodeStreamTaskUpdateMsg, &req, blen, code);
+  if (code < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosArrayDestroy(req.pNodeList);
+    return -1;
+  }
+
+  int32_t tlen = sizeof(SMsgHead) + blen;
+
+  void *buf = taosMemoryMalloc(tlen);
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosArrayDestroy(req.pNodeList);
+    return -1;
+  }
+
+  void    *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  SEncoder encoder;
+  tEncoderInit(&encoder, abuf, tlen);
+  tEncodeStreamTaskUpdateMsg(&encoder, &req);
+
+  SMsgHead *pMsgHead = (SMsgHead *)buf;
+  pMsgHead->contLen = htonl(tlen);
+  pMsgHead->vgId = htonl(nodeId);
+
+  tEncoderClear(&encoder);
+
+  *pBuf = buf;
+  *pLen = tlen;
+
+  taosArrayDestroy(req.pNodeList);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
+  void   *pBuf = NULL;
+  int32_t len = 0;
+  streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
+
+  doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
+
+  int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
+  if (code != TSDB_CODE_SUCCESS) {
+    taosMemoryFree(pBuf);
+  }
+
+  return code;
+}
+
+// build trans to update the epset
+int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
+  mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
+  taosWLockLatch(&pStream->lock);
+
+  SStreamTaskIter *pIter = createTaskIter(pStream);
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
+    int32_t      code = doSetUpdateTaskAction(pTrans, pTask, pInfo);
+    if (code != TSDB_CODE_SUCCESS) {
+      destroyTaskIter(pIter);
+      taosWUnLockLatch(&pStream->lock);
+      return -1;
+    }
+  }
+
+  destroyTaskIter(pIter);
+  taosWUnLockLatch(&pStream->lock);
+  return 0;
+}
+
+static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
+  SVResetStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq));
+  if (pReq == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq),
+           tstrerror(TSDB_CODE_OUT_OF_MEMORY));
+    return terrno;
+  }
+
+  pReq->head.vgId = htonl(pTask->info.nodeId);
+  pReq->taskId = pTask->id.taskId;
+  pReq->streamId = pTask->id.streamId;
+
+  SEpSet  epset = {0};
+  bool    hasEpset = false;
+  int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
+  if (code != TSDB_CODE_SUCCESS || !hasEpset) {
+    taosMemoryFree(pReq);
+    return code;
+  }
+
+  code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0);
+  if (code != TSDB_CODE_SUCCESS) {
+    taosMemoryFree(pReq);
+  }
+
+  return code;
+}
+
+int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
+  taosWLockLatch(&pStream->lock);
+
+  SStreamTaskIter *pIter = createTaskIter(pStream);
+  while (taskIterNextTask(pIter)) {
+    SStreamTask *pTask = taskIterGetCurrent(pIter);
+    int32_t      code = doSetResetAction(pMnode, pTrans, pTask);
+    if (code != TSDB_CODE_SUCCESS) {
+      destroyTaskIter(pIter);
+      taosWUnLockLatch(&pStream->lock);
+      return -1;
+    }
+  }
+
+  destroyTaskIter(pIter);
+  taosWUnLockLatch(&pStream->lock);
+  return 0;
+}

From 35f154da375a11997a737177f1a7ad9ee44da3a1 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 29 Jan 2024 13:55:46 +0800
Subject: [PATCH 12/33] fix:[TS-4479] support long varbinary in format like
 \x3423

---
 source/libs/parser/src/parInsertSql.c |  6 +--
 utils/test/c/varbinary_test.c         | 55 ++++++++++++++++++++++++++-
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c
index ce2d51c911..512dfdaef2 100644
--- a/source/libs/parser/src/parInsertSql.c
+++ b/source/libs/parser/src/parInsertSql.c
@@ -440,14 +440,14 @@ static int32_t parseVarbinary(SToken* pToken, uint8_t **pData, uint32_t *nData,
     return TSDB_CODE_PAR_INVALID_VARBINARY;
   }
 
-  if(isHex(pToken->z, pToken->n)){
-    if(!isValidateHex(pToken->z, pToken->n)){
+  if(isHex(pToken->z + 1, pToken->n - 2)){
+    if(!isValidateHex(pToken->z + 1, pToken->n - 2)){
       return TSDB_CODE_PAR_INVALID_VARBINARY;
     }
 
     void* data = NULL;
     uint32_t size = 0;
-    if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){
+    if(taosHex2Ascii(pToken->z + 1, pToken->n - 2, &data, &size) < 0){
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
diff --git a/utils/test/c/varbinary_test.c b/utils/test/c/varbinary_test.c
index 522a820fe8..47bacf629b 100644
--- a/utils/test/c/varbinary_test.c
+++ b/utils/test/c/varbinary_test.c
@@ -85,7 +85,6 @@ void varbinary_sql_test() {
 
   // test insert
   pRes = taos_query(taos, "insert into tb2 using stb tags (2, 'tb2_bin1', 093) values (now + 2s, 'nchar1', 892, 0.3)");
-  printf("error:%s", taos_errstr(pRes));
   ASSERT(taos_errno(pRes) != 0);
 
   pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', 0x7f829) values (now + 3s, 'nchar1', 0x7f829, 0.3)");
@@ -322,6 +321,60 @@ void varbinary_sql_test() {
   printf("%s result %s\n", __FUNCTION__, taos_errstr(pRes));
   taos_free_result(pRes);
 
+  // test insert string value '\x'
+  pRes = taos_query(taos, "insert into tb5 using stb tags (5, 'tb5_bin1', '\\\\xg') values (now + 4s, 'nchar1', '\\\\xg', 0.3)");
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "select c2,t3 from stb where t3 = '\\x5C7867'");
+  while ((row = taos_fetch_row(pRes)) != NULL) {
+    int32_t*    length = taos_fetch_lengths(pRes);
+    void*         data = NULL;
+    uint32_t      size = 0;
+    if(taosAscii2Hex(row[0], length[0], &data, &size) < 0){
+      ASSERT(0);
+    }
+
+    ASSERT(memcmp(data, "\\x5C7867", size) == 0);
+    taosMemoryFree(data);
+
+    if(taosAscii2Hex(row[1], length[1], &data, &size) < 0){
+      ASSERT(0);
+    }
+
+    ASSERT(memcmp(data, "\\x5C7867", size) == 0);
+    taosMemoryFree(data);
+  }
+  taos_free_result(pRes);
+
+  // test insert
+  char tmp [65517*2+3] = {0};
+  tmp[0] = '\\';
+  tmp[1] = 'x';
+  memset(tmp + 2, 48, 65517*2);
+
+  char sql[65517*2+3 + 256] = {0};
+
+  pRes = taos_query(taos, "create stable stb1 (ts timestamp, c2 varbinary(65517)) tags (t1 int, t2 binary(8), t3 varbinary(8))");
+  taos_free_result(pRes);
+
+  sprintf(sql, "insert into tb6 using stb1 tags (6, 'tb6_bin1', '\\\\xg') values (now + 4s, '%s')", tmp);
+  pRes = taos_query(taos, sql);
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "select c2 from tb6");
+  while ((row = taos_fetch_row(pRes)) != NULL) {
+    int32_t*    length = taos_fetch_lengths(pRes);
+    void*         data = NULL;
+    uint32_t      size = 0;
+    if(taosAscii2Hex(row[0], length[0], &data, &size) < 0){
+      ASSERT(0);
+    }
+
+    ASSERT(memcmp(data, tmp, size) == 0);
+    taosMemoryFree(data);
+  }
+  taos_free_result(pRes);
+
   taos_close(taos);
 }
 

From 6e43d74677e81d935cbe3f3e75519893b2397310 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Mon, 29 Jan 2024 13:58:58 +0800
Subject: [PATCH 13/33] update coverage data

---
 README-CN.md | 2 +-
 README.md    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README-CN.md b/README-CN.md
index 4931c0177e..06ac087859 100644
--- a/README-CN.md
+++ b/README-CN.md
@@ -12,7 +12,7 @@
 
 [![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine)
 [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
-[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
+[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
 
 简体中文 | [English](README.md) | [TDengine 云服务](https://cloud.taosdata.com/?utm_medium=cn&utm_source=github) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/)
diff --git a/README.md b/README.md
index 31d3a8bf67..e390b5e764 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 
 [![Build Status](https://cloud.drone.io/api/badges/taosdata/TDengine/status.svg?ref=refs/heads/master)](https://cloud.drone.io/taosdata/TDengine)
 [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
-[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
+[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
 <br />
 [![Twitter Follow](https://img.shields.io/twitter/follow/tdenginedb?label=TDengine&style=social)](https://twitter.com/tdenginedb)

From b9927cce1408760372f61b7168816318efaffaf3 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Mon, 29 Jan 2024 16:04:08 +0800
Subject: [PATCH 14/33] fix: count error on tag which is null

---
 source/common/src/tdatablock.c          | 1 +
 source/libs/executor/src/scanoperator.c | 4 ++--
 source/libs/function/src/builtinsimpl.c | 6 ++++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 5382259899..8739068432 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -224,6 +224,7 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
 
 int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows,
                          bool trimValue) {
+  if (currentRow >= numOfRows) return TSDB_CODE_SUCCESS;
   int32_t len = pColumnInfoData->info.bytes;
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     len = varDataTLen(pData);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 9243f385d0..1b69bcce94 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -684,7 +684,7 @@ void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) {
 static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock,
                                          const STableKeyInfo* tbInfo) {
   blockDataEmpty(pBlock);
-  pBlock->info.rows = 1;
+  pBlock->info.rows = 0;
   pBlock->info.id.uid = tbInfo->uid;
   pBlock->info.id.groupId = tbInfo->groupId;
 
@@ -696,7 +696,7 @@ static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBas
   }
 
   // set tag/tbname
-  doSetTagColumnData(pBase, pBlock, pTaskInfo, pBlock->info.rows);
+  doSetTagColumnData(pBase, pBlock, pTaskInfo, 1);
   return pBlock;
 }
 
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 000f634fe5..fc1fe29332 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -2338,6 +2338,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
     return TSDB_CODE_SUCCESS;
   }
 
+  if(pInput->totalRows == 0) return TSDB_CODE_SUCCESS;
+
   SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL;
 
   TSKEY startKey = getRowPTs(pInput->pPTS, 0);
@@ -2647,7 +2649,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
   int32_t bytes = pInputCol->info.bytes;
   pInfo->bytes = bytes;
 
-  if (IS_NULL_TYPE(type)) {
+  if (IS_NULL_TYPE(type) || 0 == pInput->totalRows) {
     return TSDB_CODE_SUCCESS;
   }
 
@@ -6022,7 +6024,7 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) {
     goto _group_key_over;
   }
 
-  if (colDataIsNull_s(pInputCol, startIndex)) {
+  if (pInputCol->pData == NULL || colDataIsNull_s(pInputCol, startIndex)) {
     pInfo->isNull = true;
     pInfo->hasResult = true;
     goto _group_key_over;

From 3730e43f0e688676f2bcbe8131ff2c5fbfe33030 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 29 Jan 2024 17:24:58 +0800
Subject: [PATCH 15/33] refactor: disable some logs.

---
 source/libs/stream/src/streamMeta.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index db71b56815..8ad20f357c 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1312,12 +1312,12 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) {
 }
 
 void streamMetaRLock(SStreamMeta* pMeta) {
-  stTrace("vgId:%d meta-rlock", pMeta->vgId);
+//  stTrace("vgId:%d meta-rlock", pMeta->vgId);
   taosThreadRwlockRdlock(&pMeta->lock);
 }
 
 void streamMetaRUnLock(SStreamMeta* pMeta) {
-  stTrace("vgId:%d meta-runlock", pMeta->vgId);
+//  stTrace("vgId:%d meta-runlock", pMeta->vgId);
   int32_t code = taosThreadRwlockUnlock(&pMeta->lock);
   if (code != TSDB_CODE_SUCCESS) {
     stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code);
@@ -1327,13 +1327,13 @@ void streamMetaRUnLock(SStreamMeta* pMeta) {
 }
 
 void streamMetaWLock(SStreamMeta* pMeta) {
-  stTrace("vgId:%d meta-wlock", pMeta->vgId);
+//  stTrace("vgId:%d meta-wlock", pMeta->vgId);
   taosThreadRwlockWrlock(&pMeta->lock);
-  stTrace("vgId:%d meta-wlock completed", pMeta->vgId);
+//  stTrace("vgId:%d meta-wlock completed", pMeta->vgId);
 }
 
 void streamMetaWUnLock(SStreamMeta* pMeta) {
-  stTrace("vgId:%d meta-wunlock", pMeta->vgId);
+//  stTrace("vgId:%d meta-wunlock", pMeta->vgId);
   taosThreadRwlockUnlock(&pMeta->lock);
 }
 

From 2231b3844d423aa27ffc07eba955b9248a15e98d Mon Sep 17 00:00:00 2001
From: wangjiaming0909 <604227650@qq.com>
Date: Tue, 30 Jan 2024 08:41:42 +0800
Subject: [PATCH 16/33] doc: add supported version for to_char/timestamp funcs

---
 docs/en/12-taos-sql/10-function.md | 4 ++++
 docs/zh/12-taos-sql/10-function.md | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md
index fbdae3445b..b4f1cf65da 100644
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@@ -491,6 +491,8 @@ TO_CHAR(ts, format_str_literal)
 
 **Description**: Convert a ts column to string as the format specified
 
+**Version**: Since ver-3.2.2.0
+
 **Return value type**: VARCHAR
 
 **Applicable column types**: TIMESTAMP
@@ -550,6 +552,8 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal)
 
 **Description**: Convert a formated timestamp string to a timestamp
 
+**Version**: Since ver-3.2.2.0
+
 **Return value type**: TIMESTAMP
 
 **Applicable column types**: VARCHAR
diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md
index 66322d55f1..0482022d95 100644
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@@ -491,6 +491,8 @@ TO_CHAR(ts, format_str_literal)
 
 **功能说明**: 将timestamp类型按照指定格式转换为字符串
 
+**版本**: ver-3.2.2.0
+
 **返回结果数据类型**: VARCHAR
 
 **应用字段**: TIMESTAMP
@@ -550,6 +552,8 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal)
 
 **功能说明**: 将字符串按照指定格式转化为时间戳.
 
+**版本**: ver-3.2.2.0
+
 **返回结果数据类型**: TIMESTAMP
 
 **应用字段**: VARCHAR

From 11c9c7d93634341bba5d8c002ec79a696371d97e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 30 Jan 2024 09:17:02 +0800
Subject: [PATCH 17/33] refactor: do some internal refactor.

---
 source/dnode/mnode/impl/inc/mndStream.h      | 80 ++++++++++----------
 source/dnode/mnode/impl/src/mndStream.c      | 42 +++++-----
 source/dnode/mnode/impl/src/mndStreamTrans.c | 16 ++--
 source/dnode/mnode/impl/src/mndStreamUtil.c  | 68 ++++++++---------
 4 files changed, 101 insertions(+), 105 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index 2778d0481e..372612274f 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -26,9 +26,17 @@ extern "C" {
 #define MND_STREAM_RESERVE_SIZE 64
 #define MND_STREAM_VER_NUMBER   4
 
+#define MND_STREAM_CREATE_NAME      "stream-create"
+#define MND_STREAM_CHECKPOINT_NAME  "stream-checkpoint"
+#define MND_STREAM_PAUSE_NAME       "stream-pause"
+#define MND_STREAM_RESUME_NAME      "stream-resume"
+#define MND_STREAM_DROP_NAME        "stream-drop"
+#define MND_STREAM_TASK_RESET_NAME  "stream-task-reset"
+#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
+
 typedef struct SStreamTransInfo {
   int64_t     startTime;
-  int64_t     streamUid;
+  int64_t     streamId;
   const char *name;
   int32_t     transId;
 } SStreamTransInfo;
@@ -41,7 +49,7 @@ typedef struct SVgroupChangeInfo {
 // time to generated the checkpoint, if now() - checkpointTs >= tsCheckpointInterval, this checkpoint will be discard
 // to avoid too many checkpoints for a taskk in the waiting list
 typedef struct SCheckpointCandEntry {
-  char *  pName;
+  char   *pName;
   int64_t streamId;
   int64_t checkpointTs;
   int64_t checkpointId;
@@ -62,6 +70,9 @@ typedef struct SStreamExecInfo {
   SHashObj        *pTransferStateStreams;
 } SStreamExecInfo;
 
+extern SStreamExecInfo         execInfo;
+typedef struct SStreamTaskIter SStreamTaskIter;
+
 typedef struct SNodeEntry {
   int32_t nodeId;
   bool    stageUpdated;  // the stage has been updated due to the leader/follower change or node reboot.
@@ -69,15 +80,11 @@ typedef struct SNodeEntry {
   int64_t hbTimestamp;   // second
 } SNodeEntry;
 
-#define MND_STREAM_CREATE_NAME      "stream-create"
-#define MND_STREAM_CHECKPOINT_NAME  "stream-checkpoint"
-#define MND_STREAM_PAUSE_NAME       "stream-pause"
-#define MND_STREAM_RESUME_NAME      "stream-resume"
-#define MND_STREAM_DROP_NAME        "stream-drop"
-#define MND_STREAM_TASK_RESET_NAME  "stream-task-reset"
-#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
-
-extern SStreamExecInfo execInfo;
+typedef struct SOrphanTask {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t nodeId;
+} SOrphanTask;
 
 int32_t     mndInitStream(SMnode *pMnode);
 void        mndCleanupStream(SMnode *pMnode);
@@ -85,49 +92,38 @@ SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);
 void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
 int32_t     mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
 int32_t     mndPersistStream(STrans *pTrans, SStreamObj *pStream);
+int32_t     mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId);
+int32_t     mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId);
+bool        mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock);
+int32_t     mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId);
 
-int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid);
-int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId);
-bool    mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamUid, const char *pTransName, bool lock);
-int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamUid);
-
-typedef struct SOrphanTask {
-  int64_t streamId;
-  int32_t taskId;
-  int32_t nodeId;
-} SOrphanTask;
-
-// for sma
-// TODO refactor
-int32_t     mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams);
-int32_t     mndGetNumOfStreamTasks(const SStreamObj *pStream);
-SArray     *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
-void        mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
-int32_t     setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
-                           int32_t retryCode);
-STrans     *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg);
-int32_t     mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
-SSdbRaw    *mndStreamActionEncode(SStreamObj *pStream);
-void        killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
-int32_t     mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
+int32_t  mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams);
+int32_t  mndGetNumOfStreamTasks(const SStreamObj *pStream);
+SArray  *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
+void     mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
+int32_t  setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
+                        int32_t retryCode);
+STrans  *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const char *name, const char *pMsg);
+int32_t  mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
+SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
+void     killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
+int32_t  mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
 
 SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId);
 int32_t     extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);
 int32_t     mndProcessStreamHb(SRpcMsg *pReq);
 void        saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode);
 int32_t     initStreamNodeList(SMnode *pMnode);
-int32_t     mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj* pStream, int8_t igUntreated);
+int32_t     mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated);
 int32_t     mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t     mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t     mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
 int32_t     mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 
-typedef struct SStreamTaskIter SStreamTaskIter;
-
-SStreamTaskIter *createTaskIter(SStreamObj *pStream);
-bool             taskIterNextTask(SStreamTaskIter *pIter);
-SStreamTask     *taskIterGetCurrent(SStreamTaskIter *pIter);
-void             destroyTaskIter(SStreamTaskIter *pIter);
+SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
+void             destroyStreamTaskIter(SStreamTaskIter *pIter);
+bool             streamTaskIterNextTask(SStreamTaskIter *pIter);
+SStreamTask     *streamTaskIterGetCurrent(SStreamTaskIter *pIter);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index c78b80d79a..48281fc010 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -476,16 +476,16 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
 }
 
 int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) {
-  SStreamTaskIter *pIter = createTaskIter(pStream);
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
     if (mndPersistTaskDeployReq(pTrans, pTask) < 0) {
-      destroyTaskIter(pIter);
+      destroyStreamTaskIter(pIter);
       return -1;
     }
   }
 
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
 
   // persistent stream task for already stored ts data
   if (pStream->conf.fillHistory) {
@@ -1506,9 +1506,9 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
     }
 
     // add row for each task
-    SStreamTaskIter *pIter = createTaskIter(pStream);
-    while (taskIterNextTask(pIter)) {
-      SStreamTask *pTask = taskIterGetCurrent(pIter);
+    SStreamTaskIter *pIter = createStreamTaskIter(pStream);
+    while (streamTaskIterNextTask(pIter)) {
+      SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
 
       int32_t code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows);
       if (code == TSDB_CODE_SUCCESS) {
@@ -1516,7 +1516,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       }
     }
 
-    destroyTaskIter(pIter);
+    destroyStreamTaskIter(pIter);
     taosRUnLockLatch(&pStream->lock);
 
     sdbRelease(pSdb, pStream);
@@ -1859,16 +1859,16 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
 
     taosWLockLatch(&pStream->lock);
 
-    SStreamTaskIter *pTaskIter = createTaskIter(pStream);
-    while (taskIterNextTask(pTaskIter)) {
-      SStreamTask *pTask = taskIterGetCurrent(pTaskIter);
+    SStreamTaskIter *pTaskIter = createStreamTaskIter(pStream);
+    while (streamTaskIterNextTask(pTaskIter)) {
+      SStreamTask *pTask = streamTaskIterGetCurrent(pTaskIter);
 
       SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId};
       epsetAssign(&entry.epset, &pTask->info.epSet);
       taosHashPut(pHash, &entry.nodeId, sizeof(entry.nodeId), &entry, sizeof(entry));
     }
 
-    destroyTaskIter(pTaskIter);
+    destroyStreamTaskIter(pTaskIter);
     taosWUnLockLatch(&pStream->lock);
 
     sdbRelease(pSdb, pStream);
@@ -2056,9 +2056,9 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
 }
 
 void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
-  SStreamTaskIter *pIter = createTaskIter(pStream);
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
 
     STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
     void   *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
@@ -2073,13 +2073,13 @@ void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
     }
   }
 
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
 }
 
 void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
-  SStreamTaskIter *pIter = createTaskIter(pStream);
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
 
     STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
     void   *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
@@ -2099,7 +2099,7 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
     }
   }
 
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
   ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
 }
 
diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c
index f8497e14f7..5bfd3933b5 100644
--- a/source/dnode/mnode/impl/src/mndStreamTrans.c
+++ b/source/dnode/mnode/impl/src/mndStreamTrans.c
@@ -23,10 +23,10 @@ typedef struct SKeyInfo {
 
 static int32_t clearFinishedTrans(SMnode* pMnode);
 
-int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid) {
+int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamId) {
   SStreamTransInfo info = {
-      .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamUid = streamUid};
-  taosHashPut(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid), &info, sizeof(SStreamTransInfo));
+      .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId};
+  taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo));
   return 0;
 }
 
@@ -65,7 +65,7 @@ int32_t clearFinishedTrans(SMnode* pMnode) {
   return 0;
 }
 
-bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char* pTransName, bool lock) {
+bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* pTransName, bool lock) {
   if (lock) {
     taosThreadMutexLock(&execInfo.lock);
   }
@@ -80,7 +80,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char*
 
   clearFinishedTrans(pMnode);
 
-  SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid));
+  SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
   if (pEntry != NULL) {
     SStreamTransInfo tInfo = *pEntry;
 
@@ -90,7 +90,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char*
 
     if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) {
       if ((strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) && (strcmp(pTransName, MND_STREAM_TASK_RESET_NAME) != 0)) {
-        mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid,
+        mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamId,
               tInfo.name);
         terrno = TSDB_CODE_MND_TRANS_CONFLICT;
         return true;
@@ -99,13 +99,13 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char*
       }
     } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) ||
                (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) {
-      mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid,
+      mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamId,
             tInfo.name);
       terrno = TSDB_CODE_MND_TRANS_CONFLICT;
       return true;
     }
   } else {
-    mDebug("stream:0x%"PRIx64" no conflict trans existed, continue create trans", streamUid);
+    mDebug("stream:0x%"PRIx64" no conflict trans existed, continue create trans", streamId);
   }
 
   if (lock) {
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index c28ba43f65..235c604b27 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -26,7 +26,7 @@ struct SStreamTaskIter {
   SStreamTask *pTask;
 };
 
-SStreamTaskIter* createTaskIter(SStreamObj* pStream) {
+SStreamTaskIter* createStreamTaskIter(SStreamObj* pStream) {
   SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
   if (pIter == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -42,7 +42,7 @@ SStreamTaskIter* createTaskIter(SStreamObj* pStream) {
   return pIter;
 }
 
-bool taskIterNextTask(SStreamTaskIter* pIter) {
+bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
   if (pIter->level >= pIter->totalLevel) {
     pIter->pTask = NULL;
     return false;
@@ -70,11 +70,11 @@ bool taskIterNextTask(SStreamTaskIter* pIter) {
   return false;
 }
 
-SStreamTask* taskIterGetCurrent(SStreamTaskIter* pIter) {
+SStreamTask* streamTaskIterGetCurrent(SStreamTaskIter* pIter) {
   return pIter->pTask;
 }
 
-void destroyTaskIter(SStreamTaskIter* pIter) {
+void destroyStreamTaskIter(SStreamTaskIter* pIter) {
   taosMemoryFree(pIter);
 }
 
@@ -235,16 +235,16 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
 }
 
 SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) {
-  SStreamTaskIter *pIter = createTaskIter(pStream);
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
     if (pTask->id.taskId == pId->taskId) {
-      destroyTaskIter(pIter);
+      destroyStreamTaskIter(pIter);
       return pTask;
     }
   }
 
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
   return NULL;
 }
 
@@ -259,12 +259,12 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
 }
 
 int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) {
-  SStreamTaskIter *pIter = createTaskIter(pStream);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
 
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
     if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) {
-      destroyTaskIter(pIter);
+      destroyStreamTaskIter(pIter);
       return -1;
     }
 
@@ -272,7 +272,7 @@ int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pSt
       atomic_store_8(&pTask->status.taskStatus, pTask->status.statusBackup);
     }
   }
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
   return 0;
 }
 
@@ -308,12 +308,12 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
 }
 
 int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
-  SStreamTaskIter *pIter = createTaskIter(pStream);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
 
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
     if (doSetPauseAction(pMnode, pTrans, pTask) < 0) {
-      destroyTaskIter(pIter);
+      destroyStreamTaskIter(pIter);
       return -1;
     }
 
@@ -323,7 +323,7 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr
     }
   }
 
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
   return 0;
 }
 
@@ -357,16 +357,16 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
 }
 
 int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
-  SStreamTaskIter *pIter = createTaskIter(pStream);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
 
-  while(taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  while(streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
     if (doSetDropAction(pMnode, pTrans, pTask) < 0) {
-      destroyTaskIter(pIter);
+      destroyStreamTaskIter(pIter);
       return -1;
     }
   }
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
   return 0;
 }
 
@@ -480,18 +480,18 @@ int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pI
   mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
   taosWLockLatch(&pStream->lock);
 
-  SStreamTaskIter *pIter = createTaskIter(pStream);
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
     int32_t      code = doSetUpdateTaskAction(pTrans, pTask, pInfo);
     if (code != TSDB_CODE_SUCCESS) {
-      destroyTaskIter(pIter);
+      destroyStreamTaskIter(pIter);
       taosWUnLockLatch(&pStream->lock);
       return -1;
     }
   }
 
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
   taosWUnLockLatch(&pStream->lock);
   return 0;
 }
@@ -528,18 +528,18 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
 int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
   taosWLockLatch(&pStream->lock);
 
-  SStreamTaskIter *pIter = createTaskIter(pStream);
-  while (taskIterNextTask(pIter)) {
-    SStreamTask *pTask = taskIterGetCurrent(pIter);
+  SStreamTaskIter *pIter = createStreamTaskIter(pStream);
+  while (streamTaskIterNextTask(pIter)) {
+    SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
     int32_t      code = doSetResetAction(pMnode, pTrans, pTask);
     if (code != TSDB_CODE_SUCCESS) {
-      destroyTaskIter(pIter);
+      destroyStreamTaskIter(pIter);
       taosWUnLockLatch(&pStream->lock);
       return -1;
     }
   }
 
-  destroyTaskIter(pIter);
+  destroyStreamTaskIter(pIter);
   taosWUnLockLatch(&pStream->lock);
   return 0;
 }

From d66d5335bb791860bcec0e69dd6a55c51673937d Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 30 Jan 2024 09:38:14 +0800
Subject: [PATCH 18/33] fix:python error

---
 tests/system-test/0-others/subscribe_stream_privilege.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py
index d85d87dc3a..b477af9f57 100644
--- a/tests/system-test/0-others/subscribe_stream_privilege.py
+++ b/tests/system-test/0-others/subscribe_stream_privilege.py
@@ -129,10 +129,10 @@ class TDTestCase:
         if not exceptOccured:
             tdLog.exit(f"has no privilege, should except")
 
-        checkUserPrivileges(1)
+        self.checkUserPrivileges(1)
         tdLog.debug("test subscribe topic privilege granted by other user")
         tdSql.execute(f'grant subscribe on {self.topic_name} to {self.user_name}')
-        checkUserPrivileges(2)
+        self.checkUserPrivileges(2)
 
         exceptOccured = False
         try:
@@ -159,7 +159,7 @@ class TDTestCase:
 
                 tdLog.debug("test subscribe topic privilege revoked by other user")
                 tdSql.execute(f'revoke subscribe on {self.topic_name} from {self.user_name}')
-                checkUserPrivileges(1)
+                self.checkUserPrivileges(1)
                 time.sleep(5)
 
         finally:

From 49700404593568ede97c1b11bbe899bd327a3dcc Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 30 Jan 2024 15:22:49 +0800
Subject: [PATCH 19/33] refactor: do some internal refactor.

---
 include/libs/stream/tstream.h        |  9 +++++----
 source/dnode/vnode/src/tq/tq.c       |  4 ++--
 source/libs/executor/src/executor.c  |  2 +-
 source/libs/stream/src/streamExec.c  | 10 ++++++++--
 source/libs/stream/src/streamMeta.c  |  2 +-
 source/libs/stream/src/streamStart.c |  2 --
 source/libs/stream/src/streamState.c |  1 -
 7 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 9b3ce36bdd..747ba34c97 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -324,12 +324,13 @@ typedef struct SStreamStatus {
   int8_t         taskStatus;
   int8_t         downstreamReady;  // downstream tasks are all ready now, if this flag is set
   int8_t         schedStatus;
-  int32_t        schedIdleTime;    // idle time before invoke again
-  int64_t        lastExecTs;       // last exec time stamp
   int8_t         statusBackup;
-  bool           appendTranstateBlock;  // has append the transfer state data block already
-  int32_t        timerActive;           // timer is active
+  int32_t        schedIdleTime;  // idle time before invoke again
+  int32_t        timerActive;    // timer is active
+  int64_t        lastExecTs;     // last exec time stamp
   int32_t        inScanHistorySentinel;
+  bool           appendTranstateBlock;  // has append the transfer state data block already
+  bool           supplementaryWalscan;  // complete the supplementary wal scan or not
 } SStreamStatus;
 
 typedef struct SDataRange {
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index a689932754..1ade1c8c41 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -835,8 +835,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   SCheckpointInfo* pChkInfo = &pTask->chkInfo;
 
   // checkpoint ver is the kept version, handled data should be the next version.
-  if (pTask->chkInfo.checkpointId != 0) {
-    pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1;
+  if (pChkInfo->checkpointId != 0) {
+    pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1;
     tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr,
            pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
   }
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index e1a8e8ea01..9645fc1b7a 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1080,7 +1080,7 @@ bool qStreamScanhistoryFinished(qTaskInfo_t tinfo) {
 
 int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  STimeWindow* pWindow = &pTaskInfo->streamInfo.fillHistoryWindow;
+  STimeWindow*   pWindow = &pTaskInfo->streamInfo.fillHistoryWindow;
 
   qDebug("%s remove timeWindow filter:%" PRId64 "-%" PRId64 ", set new window:%" PRId64 "-%" PRId64,
          GET_TASKID(pTaskInfo), pWindow->skey, pWindow->ekey, INT64_MIN, INT64_MAX);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 27748c84a0..0c56679c14 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -390,6 +390,8 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
 
     pTimeWindow->skey = INT64_MIN;
     qStreamInfoResetTimewindowFilter(pStreamTask->exec.pExecutor);
+    stDebug("s-task:%s after exceed the threshold:%" PRId64 " and then update the window filter",
+            pStreamTask->id.idStr, pStreamTask->dataRange.range.maxVer);
   } else {
     stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr);
   }
@@ -398,9 +400,11 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
   streamTaskReleaseState(pTask);
   streamTaskReloadState(pStreamTask);
 
+  // 3. scan wal file from the beginning till the end version of fill-history task.
+  streamTaskSupplementaryScan(pStreamTask);
+
   // 3. send msg to mnode to launch a checkpoint to keep the state for current stream
-  streamTaskSendCheckpointReq(pStreamTask);
-//  streamTaskResume(pStreamTask);
+//  streamTaskSendCheckpointReq(pStreamTask);
 
   // 4. assign the status to the value that will be kept in disk
   pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask)->state;
@@ -777,6 +781,8 @@ int32_t streamResumeTask(SStreamTask* pTask) {
 
   while (1) {
     /*int32_t code = */ doStreamExecTask(pTask);
+
+    // check if continue
     taosThreadMutexLock(&pTask->lock);
 
     int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 8ad20f357c..5e53a921b9 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1322,7 +1322,7 @@ void streamMetaRUnLock(SStreamMeta* pMeta) {
   if (code != TSDB_CODE_SUCCESS) {
     stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code);
   } else {
-    stDebug("vgId:%d meta-runlock completed", pMeta->vgId);
+//    stTrace("vgId:%d meta-runlock completed", pMeta->vgId);
   }
 }
 
diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c
index 20fdcff7d9..2f5bca8ed9 100644
--- a/source/libs/stream/src/streamStart.c
+++ b/source/libs/stream/src/streamStart.c
@@ -385,7 +385,6 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) {
 
 void doProcessDownstreamReadyRsp(SStreamTask* pTask) {
   EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
-
   streamTaskOnHandleEventSuccess(pTask->status.pSM, event);
 
   int64_t initTs = pTask->execInfo.init;
@@ -989,4 +988,3 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
     streamSetParamForStreamScannerStep2(pTask, &verRange, &win);
   }
 }
-
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 19b7359981..84b0a777f2 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -670,7 +670,6 @@ void streamStateFreeCur(SStreamStateCur* pCur) {
   if (!pCur) {
     return;
   }
-  qDebug("streamStateFreeCur");
   streamStateResetCur(pCur);
   taosMemoryFree(pCur);
 }

From 1d110953c9bc091f2181b9def8b7f6018161c68f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 30 Jan 2024 15:23:54 +0800
Subject: [PATCH 20/33] refactor: do some internal refactor.

---
 source/libs/stream/src/streamExec.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 0c56679c14..b0170d5083 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -400,11 +400,8 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
   streamTaskReleaseState(pTask);
   streamTaskReloadState(pStreamTask);
 
-  // 3. scan wal file from the beginning till the end version of fill-history task.
-  streamTaskSupplementaryScan(pStreamTask);
-
   // 3. send msg to mnode to launch a checkpoint to keep the state for current stream
-//  streamTaskSendCheckpointReq(pStreamTask);
+  streamTaskSendCheckpointReq(pStreamTask);
 
   // 4. assign the status to the value that will be kept in disk
   pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask)->state;

From efbc7e14024aa09f5ffd5d52f079596393d3cb23 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 30 Jan 2024 15:35:01 +0800
Subject: [PATCH 21/33] fix(stream): fix memory leak.

---
 source/dnode/mnode/impl/src/mndStreamHb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c
index 19c4f22280..ff0d8b82fc 100644
--- a/source/dnode/mnode/impl/src/mndStreamHb.c
+++ b/source/dnode/mnode/impl/src/mndStreamHb.c
@@ -183,6 +183,7 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
     return -1;
   }
 
+  mndTransDrop(pTrans);
   return 0;
 }
 

From 5d731dc9a4d039acf72baeaa1833be4742945e3d Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Tue, 30 Jan 2024 17:34:06 +0800
Subject: [PATCH 22/33] fix: use blank data flag

---
 include/common/tcommon.h                | 1 +
 include/libs/function/function.h        | 1 +
 source/common/src/tdatablock.c          | 1 -
 source/libs/executor/src/executorInt.c  | 2 ++
 source/libs/executor/src/scanoperator.c | 3 ++-
 source/libs/function/src/builtinsimpl.c | 7 ++++---
 6 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 24e5d186b9..d4537ddc89 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -206,6 +206,7 @@ typedef struct SDataBlockInfo {
   int16_t     hasVarCol;
   int16_t     dataLoad;  // denote if the data is loaded or not
   uint8_t     scanFlag;
+  bool        blankFill;
 
   // TODO: optimize and remove following
   int64_t     version;    // used for stream, and need serialization
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 8863201094..0fa84c99c6 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -114,6 +114,7 @@ typedef struct SInputColumnInfoData {
   int32_t           totalRows;        // total rows in current columnar data
   int32_t           startRowIndex;    // handle started row index
   int64_t           numOfRows;        // the number of rows needs to be handled
+  bool              blankFill;        // fill blank data to block for empty table
   int32_t           numOfInputCols;   // PTS is not included
   bool              colDataSMAIsSet;  // if agg is set or not
   SColumnInfoData  *pPTS;             // primary timestamp column
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 8739068432..5382259899 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -224,7 +224,6 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
 
 int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows,
                          bool trimValue) {
-  if (currentRow >= numOfRows) return TSDB_CODE_SUCCESS;
   int32_t len = pColumnInfoData->info.bytes;
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     len = varDataTLen(pData);
diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c
index ff4d3d0d27..76dc622cfd 100644
--- a/source/libs/executor/src/executorInt.c
+++ b/source/libs/executor/src/executorInt.c
@@ -311,6 +311,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
         pInput->totalRows = pBlock->info.rows;
         pInput->numOfRows = pBlock->info.rows;
         pInput->startRowIndex = 0;
+        pInput->blankFill = pBlock->info.blankFill;
 
         // NOTE: the last parameter is the primary timestamp column
         // todo: refactor this
@@ -325,6 +326,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
           pInput->totalRows = pBlock->info.rows;
           pInput->numOfRows = pBlock->info.rows;
           pInput->startRowIndex = 0;
+          pInput->blankFill = pBlock->info.blankFill;
 
           code = doCreateConstantValColumnInfo(pInput, pFuncParam, j, pBlock->info.rows);
           if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1b69bcce94..1299638935 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -684,9 +684,10 @@ void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) {
 static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock,
                                          const STableKeyInfo* tbInfo) {
   blockDataEmpty(pBlock);
-  pBlock->info.rows = 0;
+  pBlock->info.rows = 1;
   pBlock->info.id.uid = tbInfo->uid;
   pBlock->info.id.groupId = tbInfo->groupId;
+  pBlock->info.blankFill = true;
 
   // only one row: set all col data to null & hasNull
   int32_t col_num = blockDataGetNumOfCols(pBlock);
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index fc1fe29332..5ab6d5e075 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -499,6 +499,9 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) {
    */
   SInputColumnInfoData* pInput = &pCtx->input;
   SColumnInfoData*      pInputCol = pInput->pData[0];
+  if(1 == pInput->numOfRows && pInput->blankFill) {
+    return 0;
+  }
   if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) {
     numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull;
   } else {
@@ -2338,8 +2341,6 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if(pInput->totalRows == 0) return TSDB_CODE_SUCCESS;
-
   SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL;
 
   TSKEY startKey = getRowPTs(pInput->pPTS, 0);
@@ -2649,7 +2650,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
   int32_t bytes = pInputCol->info.bytes;
   pInfo->bytes = bytes;
 
-  if (IS_NULL_TYPE(type) || 0 == pInput->totalRows) {
+  if (IS_NULL_TYPE(type)) {
     return TSDB_CODE_SUCCESS;
   }
 

From 917a6eb5a7861e97ae4ce5d2ed8b71781a0c9a38 Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Tue, 30 Jan 2024 09:47:20 +0000
Subject: [PATCH 23/33] fix/TD-28503

---
 source/dnode/mnode/impl/src/mndCompact.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c
index 101022a44f..4e71684372 100644
--- a/source/dnode/mnode/impl/src/mndCompact.c
+++ b/source/dnode/mnode/impl/src/mndCompact.c
@@ -599,7 +599,8 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
             pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
             pDetail->newNumberFileset, pDetail->newFinished);
 
-      if(pDetail->numberFileset < pDetail->newNumberFileset || pDetail->finished < pDetail->newFinished)
+      //these 2 number will jump back after dnode restart, so < is not used here
+      if(pDetail->numberFileset != pDetail->newNumberFileset || pDetail->finished != pDetail->newFinished)
         needSave = true;
     }
 

From 9c0d2e603a9e649c0a6699202d8990c844f75078 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Tue, 30 Jan 2024 20:24:59 +0800
Subject: [PATCH 24/33] reset blankFill

---
 source/libs/executor/src/scanoperator.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1299638935..bbe12c345e 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -663,6 +663,8 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i
 
   pInfo->tableEndIndex = (pInfo->tableStartIndex + (*size) - 1);
 
+  pInfo->pResBlock->info.blankFill = false;
+
   if (!pInfo->needCountEmptyTable) {
     pInfo->countState = TABLE_COUNT_STATE_END;
   } else {

From 83950025e51612093c3398d1150b57d07e8600a6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 31 Jan 2024 08:58:17 +0800
Subject: [PATCH 25/33] fix(stream): limit the available threads for stream
 processing.

---
 source/common/src/tglobal.c | 2 +-
 source/util/src/tworker.c   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index a7d80fe5db..3b965553af 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
-float   tsRatioOfVnodeStreamThreads = 1.0;
+float   tsRatioOfVnodeStreamThreads = 0.5;
 int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeRsmaThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 4;
diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c
index 57dc60e539..28f7fd4783 100644
--- a/source/util/src/tworker.c
+++ b/source/util/src/tworker.c
@@ -221,7 +221,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem
   int32_t queueNum = taosGetQueueNumber(pool->qset);
   int32_t curWorkerNum = taosArrayGetSize(pool->workers);
   int32_t dstWorkerNum = ceilf(queueNum * pool->ratio);
-  if (dstWorkerNum < 1) dstWorkerNum = 1;
+  if (dstWorkerNum < 2) dstWorkerNum = 2;
 
   // spawn a thread to process queue
   while (curWorkerNum < dstWorkerNum) {

From 2716119c92b7b8884768aaca7c4c9f94240c8437 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 31 Jan 2024 09:00:26 +0800
Subject: [PATCH 26/33] fix(other): update the upper limit for stream threads.

---
 source/common/src/tglobal.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 3b965553af..53169019b6 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
-float   tsRatioOfVnodeStreamThreads = 0.5;
+float   tsRatioOfVnodeStreamThreads = 0.5F;
 int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeRsmaThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 4;
@@ -622,7 +622,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
       0)
     return -1;
 
-  if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 10, CFG_SCOPE_SERVER,
+  if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER,
                   CFG_DYN_NONE) != 0)
     return -1;
 

From 5cf8687fd7cc7f68ee5554d34d1693b9d50e46b7 Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Wed, 31 Jan 2024 04:23:28 +0000
Subject: [PATCH 27/33] fix/TD-28519

---
 source/dnode/mnode/impl/src/mndTrans.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 9075f0145c..1adc4ed4bf 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -599,6 +599,8 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
     pTrans->originRpcType = pReq->msgType;
   }
 
+  mInfo("trans:%d, create transaction:%s, origin:%s", pTrans->id, pTrans->opername, opername);
+
   mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans);
   return pTrans;
 }
@@ -845,6 +847,8 @@ int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) {
 }
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
+  if(pTrans == NULL) return -1;
+
   if (mndTransCheckConflict(pMnode, pTrans) != 0) {
     return -1;
   }

From 89bcc8f0582c03fe5fea69717cc8880bbef2a75c Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Wed, 31 Jan 2024 07:10:01 +0000
Subject: [PATCH 28/33] add queue perf msg

---
 source/libs/transport/src/transSvr.c |  4 ++--
 source/util/src/tqueue.c             |  6 ++++--
 source/util/src/tworker.c            | 25 ++++++++++++++++++++++++-
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 362d38b505..b324ca5f91 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg);
 
 static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
 
-static FORCE_INLINE void destroySmsg(SSvrMsg* smsg);
+static FORCE_INLINE void      destroySmsg(SSvrMsg* smsg);
 static FORCE_INLINE SSvrConn* createConn(void* hThrd);
 static FORCE_INLINE void      destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
 static FORCE_INLINE void      destroyConnRegArg(SSvrConn* conn);
@@ -1382,7 +1382,7 @@ void uvHandleUpdate(SSvrMsg* msg, SWorkThrd* thrd) {
     tFreeSUpdateIpWhiteReq(req);
     taosMemoryFree(req);
   } else {
-    tInfo("ip-white-list disable on trans");
+    tDebug("ip-white-list disable on trans");
     thrd->enableIpWhiteList = 0;
   }
   taosMemoryFree(msg);
diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c
index 1dfdd637b6..2cc13be6ba 100644
--- a/source/util/src/tqueue.c
+++ b/source/util/src/tqueue.c
@@ -159,6 +159,7 @@ void taosFreeQitem(void *pItem) {
 int32_t taosWriteQitem(STaosQueue *queue, void *pItem) {
   int32_t     code = 0;
   STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode));
+  pNode->timestamp = taosGetTimestampUs();
   pNode->next = NULL;
 
   taosThreadMutexLock(&queue->mutex);
@@ -464,6 +465,7 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *
       qinfo->ahandle = queue->ahandle;
       qinfo->fp = queue->itemsFp;
       qinfo->queue = queue;
+      qinfo->timestamp = queue->head->timestamp;
 
       queue->head = NULL;
       queue->tail = NULL;
@@ -489,8 +491,8 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *
 int32_t taosQallItemSize(STaosQall *qall) { return qall->numOfItems; }
 int64_t taosQallMemSize(STaosQall *qall) { return qall->memOfItems; }
 
-int64_t taosQallUnAccessedItemSize(STaosQall *qall) {return qall->unAccessedNumOfItems;}
-int64_t taosQallUnAccessedMemSize(STaosQall *qall) {return qall->unAccessMemOfItems;}
+int64_t taosQallUnAccessedItemSize(STaosQall *qall) { return qall->unAccessedNumOfItems; }
+int64_t taosQallUnAccessedMemSize(STaosQall *qall) { return qall->unAccessMemOfItems; }
 
 void    taosResetQitems(STaosQall *qall) { qall->current = qall->start; }
 int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; }
diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c
index 28f7fd4783..c4b3271c65 100644
--- a/source/util/src/tworker.c
+++ b/source/util/src/tworker.c
@@ -15,10 +15,12 @@
 
 #define _DEFAULT_SOURCE
 #include "tworker.h"
-#include "tgeosctx.h"
 #include "taoserror.h"
+#include "tgeosctx.h"
 #include "tlog.h"
 
+#define QUEUE_THRESHOLD 1000 * 1000
+
 typedef void *(*ThreadFp)(void *param);
 
 int32_t tQWorkerInit(SQWorkerPool *pool) {
@@ -84,6 +86,13 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) {
       break;
     }
 
+    if (qinfo.timestamp != 0) {
+      int64_t cost = taosGetTimestampUs() - qinfo.timestamp;
+      if (cost > QUEUE_THRESHOLD) {
+        uWarn("worker:%s,message has been queued for too long, cost: %" PRId64 "s", pool->name, cost);
+      }
+    }
+
     if (qinfo.fp != NULL) {
       qinfo.workerId = worker->id;
       qinfo.threadNum = pool->num;
@@ -198,6 +207,13 @@ static void *tAutoQWorkerThreadFp(SQueueWorker *worker) {
       break;
     }
 
+    if (qinfo.timestamp != 0) {
+      int64_t cost = taosGetTimestampUs() - qinfo.timestamp;
+      if (cost > QUEUE_THRESHOLD) {
+        uWarn("worker:%s,message has been queued for too long, cost: %" PRId64 "s", pool->name, cost);
+      }
+    }
+
     if (qinfo.fp != NULL) {
       qinfo.workerId = worker->id;
       qinfo.threadNum = taosArrayGetSize(pool->workers);
@@ -338,6 +354,13 @@ static void *tWWorkerThreadFp(SWWorker *worker) {
       break;
     }
 
+    if (qinfo.timestamp != 0) {
+      int64_t cost = taosGetTimestampUs() - qinfo.timestamp;
+      if (cost > QUEUE_THRESHOLD) {
+        uWarn("worker:%s,message has been queued for too long, cost: %" PRId64 "s", pool->name, cost);
+      }
+    }
+
     if (qinfo.fp != NULL) {
       qinfo.workerId = worker->id;
       qinfo.threadNum = pool->num;

From be38b589704af2dbe0a0cc59003640b78316806e Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Wed, 31 Jan 2024 16:15:30 +0800
Subject: [PATCH 29/33] fix: don't wait when tsort close

---
 source/libs/executor/src/tsort.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index db9266cb8f..3559e57ebc 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1334,7 +1334,6 @@ static bool tsortOpenForBufMergeSort(SSortHandle* pHandle) {
 
 int32_t tsortClose(SSortHandle* pHandle) {
   atomic_val_compare_exchange_8(&pHandle->closed, 0, 1);
-  taosMsleep(10);
   return TSDB_CODE_SUCCESS;
 }
 

From 74561a66c32bfdf399b09b6df33ae90da1ae08dd Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 31 Jan 2024 17:10:31 +0800
Subject: [PATCH 30/33] fix:set log level to 135 for some cases

---
 tests/parallel_test/cases.task               | 1 +
 tests/system-test/7-tmq/tmqDropConsumer.py   | 3 ++-
 tests/system-test/7-tmq/tmqParamsTest.py     | 4 +++-
 tests/system-test/7-tmq/tmqVnodeTransform.py | 6 +++---
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 9bc2827157..91a0ac46e5 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -230,6 +230,7 @@ fi
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True
+,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1
 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1
 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3
 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3
diff --git a/tests/system-test/7-tmq/tmqDropConsumer.py b/tests/system-test/7-tmq/tmqDropConsumer.py
index 5208d14069..953e9314f1 100644
--- a/tests/system-test/7-tmq/tmqDropConsumer.py
+++ b/tests/system-test/7-tmq/tmqDropConsumer.py
@@ -12,7 +12,8 @@ sys.path.append("./7-tmq")
 from tmqCommon import *
 
 class TDTestCase:
-    # updatecfgDict = {'debugFlag': 135}
+    clientCfgDict = {'debugFlag': 135}
+    updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict}
     
     def __init__(self):
         self.vgroups    = 2
diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py
index 0e9e8f989f..9286b69278 100644
--- a/tests/system-test/7-tmq/tmqParamsTest.py
+++ b/tests/system-test/7-tmq/tmqParamsTest.py
@@ -11,7 +11,9 @@ sys.path.append("./7-tmq")
 from tmqCommon import *
 
 class TDTestCase:
-    updatecfgDict = {'debugFlag': 135}
+    clientCfgDict = {'debugFlag': 135}
+    updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict}
+
     def init(self, conn, logSql, replicaVar=1):
         self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py
index 811b72c35f..c2b002ead6 100644
--- a/tests/system-test/7-tmq/tmqVnodeTransform.py
+++ b/tests/system-test/7-tmq/tmqVnodeTransform.py
@@ -186,7 +186,7 @@ class TDTestCase:
         tmqCom.getStartCommitNotifyFromTmqsim()
 
         #restart dnode & remove wal
-        # self.restartAndRemoveWal()
+        self.restartAndRemoveWal()
 
         # redistribute vgroup
         self.redistributeVgroups();
@@ -235,7 +235,7 @@ class TDTestCase:
         tdSql.execute(sqlString)
         tdSql.query("flush database %s"%(paraDict['dbName']))
         #restart dnode & remove wal
-        # self.restartAndRemoveWal()
+        self.restartAndRemoveWal()
 
         # redistribute vgroup
         self.redistributeVgroups();
@@ -313,7 +313,7 @@ class TDTestCase:
 
         time.sleep(5)
         #restart dnode & remove wal
-        # self.restartAndRemoveWal()
+        self.restartAndRemoveWal()
 
         # redistribute vgroup
         self.redistributeVgroups()

From 318d23c3003218faaa2c2312d567216120650c8b Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 31 Jan 2024 17:27:07 +0800
Subject: [PATCH 31/33] docs:add description for \x

---
 docs/en/12-taos-sql/18-escape.md | 4 ++--
 docs/zh/12-taos-sql/18-escape.md | 5 ++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/docs/en/12-taos-sql/18-escape.md b/docs/en/12-taos-sql/18-escape.md
index a44b21db43..6279b3e809 100644
--- a/docs/en/12-taos-sql/18-escape.md
+++ b/docs/en/12-taos-sql/18-escape.md
@@ -22,5 +22,5 @@ description: This document describes the usage of escape characters in TDengine.
    - Identifier without ``: Error will be returned because identifier must be constituted of digits, ASCII characters or underscore and can't be started with digits
    - Identifier quoted with ``: Original content is kept, no escaping
 2. If there are escape characters in values
-   - The escape characters will be escaped as the above table. If the escape character doesn't match any supported one, the escape character "\" will be ignored.
-   - "%" and "\_" are used as wildcards in `like`. `\%` and `\_` should be used to represent literal "%" and "\_" in `like`,. If `\%` and `\_` are used out of `like` context, the evaluation result is "`\%`"and "`\_`", instead of "%" and "\_".
+   - The escape characters will be escaped as the above table. If the escape character doesn't match any supported one, the escape character \ will be ignored(\x remaining).
+   - "%" and "\_" are used as wildcards in `like`. `\%` and `\_` should be used to represent literal "%" and "\_" in `like`. If `\%` and `\_` are used out of `like` context, the evaluation result is "`\%`"and "`\_`", instead of "%" and "\_".
diff --git a/docs/zh/12-taos-sql/18-escape.md b/docs/zh/12-taos-sql/18-escape.md
index 5e0d292d39..eb04acec08 100644
--- a/docs/zh/12-taos-sql/18-escape.md
+++ b/docs/zh/12-taos-sql/18-escape.md
@@ -17,7 +17,6 @@ description: TDengine 中使用转义字符的详细规则
 |   `\%`   | % 规则见下     |
 |   `\_`   | \_ 规则见下    |
 
-:::
 
 ## 转义字符使用规则
 
@@ -25,5 +24,5 @@ description: TDengine 中使用转义字符的详细规则
    1. 普通标识符： 直接提示错误的标识符，因为标识符规定必须是数字、字母和下划线，并且不能以数字开头。
    2. 反引号``标识符： 保持原样，不转义
 2. 数据里有转义字符
-   1. 遇到上面定义的转义字符会转义（%和\_见下面说明），如果没有匹配的转义字符会忽略掉转义符\。
-   2. 对于%和\_，因为在 like 里这两个字符是通配符，所以在模式匹配 like 里用`\%`%和`\_`表示字符里本身的%和\_，如果在 like 模式匹配上下文之外使用`\%`或`\_`，则它们的计算结果为字符串`\%`和`\_`，而不是%和\_。
+   1. 遇到上面定义的转义字符会转义（%和\_见下面说明），如果没有匹配的转义字符会忽略掉转义符\（\x保持原样）。
+   2. 对于%和\_，因为在 like 里这两个字符是通配符，所以在模式匹配 like 里用`\%`和`\_`表示字符里本身的%和\_，如果在 like 模式匹配上下文之外使用`\%`或`\_`，则它们的计算结果为字符串`\%`和`\_`，而不是%和\_。

From 0ac01ae1b693d9cd5c36e9a96664fbab4434ea47 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 31 Jan 2024 17:37:59 +0800
Subject: [PATCH 32/33] docs:add description for \x

---
 docs/en/12-taos-sql/18-escape.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/en/12-taos-sql/18-escape.md b/docs/en/12-taos-sql/18-escape.md
index 6279b3e809..2d067b2ad9 100644
--- a/docs/en/12-taos-sql/18-escape.md
+++ b/docs/en/12-taos-sql/18-escape.md
@@ -7,14 +7,14 @@ description: This document describes the usage of escape characters in TDengine.
 
 | Escape Character | **Actual Meaning**       |
 | :--------------: | ------------------------ |
-|       `\'`       | Single quote '           |
-|       `\"`       | Double quote "           |
-|        \n        | Line Break               |
-|        \r        | Carriage Return          |
-|        \t        | tab                      |
-|       `\\`       | Back Slash \             |
-|       `\%`       | % see below for details  |
-|       `\_`       | \_ see below for details |
+|       `\'`       | Single quote `'`           |
+|       `\"`       | Double quote `"`           |
+|       `\n`        | Line Break               |
+|       `\r`        | Carriage Return          |
+|       `\t`        | tab                      |
+|       `\\`       | Back Slash `\ `             |
+|       `\%`       | `%` see below for details  |
+|       `\_`       | `_` see below for details |
 
 ## Restrictions
 
@@ -22,5 +22,5 @@ description: This document describes the usage of escape characters in TDengine.
    - Identifier without ``: Error will be returned because identifier must be constituted of digits, ASCII characters or underscore and can't be started with digits
    - Identifier quoted with ``: Original content is kept, no escaping
 2. If there are escape characters in values
-   - The escape characters will be escaped as the above table. If the escape character doesn't match any supported one, the escape character \ will be ignored(\x remaining).
-   - "%" and "\_" are used as wildcards in `like`. `\%` and `\_` should be used to represent literal "%" and "\_" in `like`. If `\%` and `\_` are used out of `like` context, the evaluation result is "`\%`"and "`\_`", instead of "%" and "\_".
+   - The escape characters will be escaped as the above table. If the escape character doesn't match any supported one, the escape character `\ ` will be ignored(`\x` remaining).
+   - `%` and `_` are used as wildcards in `like`. `\%` and `\_` should be used to represent literal `%` and `_` in `like`. If `\%` and `\_` are used out of `like` context, the evaluation result is `\%` and `\_`, instead of `%` and `_`.

From a0b3214eed05f8ec9757a2a949df654fbdfe8061 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 31 Jan 2024 17:41:10 +0800
Subject: [PATCH 33/33] docs:add description for \x

---
 docs/zh/12-taos-sql/18-escape.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/zh/12-taos-sql/18-escape.md b/docs/zh/12-taos-sql/18-escape.md
index eb04acec08..81e4179042 100644
--- a/docs/zh/12-taos-sql/18-escape.md
+++ b/docs/zh/12-taos-sql/18-escape.md
@@ -8,14 +8,14 @@ description: TDengine 中使用转义字符的详细规则
 
 | 字符序列 | **代表的字符** |
 | :------: | -------------- |
-|   `\'`   | 单引号'        |
-|   `\"`   | 双引号"        |
-|    \n    | 换行符         |
-|    \r    | 回车符         |
-|    \t    | tab 符         |
-|   `\\`   | 斜杠\          |
-|   `\%`   | % 规则见下     |
-|   `\_`   | \_ 规则见下    |
+|   `\'`   | 单引号`'`        |
+|   `\"`   | 双引号`"`        |
+|   `\n`    | 换行符         |
+|   `\r`    | 回车符         |
+|   `\t`    | tab 符         |
+|   `\\`   | 斜杠 `\ `          |
+|   `\%`   | `%` 规则见下     |
+|   `\_`   | `_` 规则见下    |
 
 
 ## 转义字符使用规则
@@ -24,5 +24,5 @@ description: TDengine 中使用转义字符的详细规则
    1. 普通标识符： 直接提示错误的标识符，因为标识符规定必须是数字、字母和下划线，并且不能以数字开头。
    2. 反引号``标识符： 保持原样，不转义
 2. 数据里有转义字符
-   1. 遇到上面定义的转义字符会转义（%和\_见下面说明），如果没有匹配的转义字符会忽略掉转义符\（\x保持原样）。
-   2. 对于%和\_，因为在 like 里这两个字符是通配符，所以在模式匹配 like 里用`\%`和`\_`表示字符里本身的%和\_，如果在 like 模式匹配上下文之外使用`\%`或`\_`，则它们的计算结果为字符串`\%`和`\_`，而不是%和\_。
+   1. 遇到上面定义的转义字符会转义（`%`和`_`见下面说明），如果没有匹配的转义字符会忽略掉转义符`\ `（`\x`保持原样）。
+   2. 对于`%`和`_`，因为在`like`里这两个字符是通配符，所以在模式匹配`like`里用`\%`和`\_`表示字符里本身的`%`和`_`，如果在`like`模式匹配上下文之外使用`\%`或`\_`，则它们的计算结果为字符串`\%`和`\_`，而不是`%`和`_`。