From 7f93ec2c5343efe6a9e932520698de9c603e2cfb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 11 May 2024 09:37:02 +0800
Subject: [PATCH 001/132] fix(stream): update the checkpoint info for
 followers, and some internal refactor.

---
 include/common/tmsg.h                         |  23 +--
 include/common/tmsgdef.h                      |   3 +-
 include/dnode/vnode/tqCommon.h                |   1 +
 include/libs/stream/tstream.h                 |   3 +
 source/common/src/rsync.c                     |   4 +-
 source/dnode/mgmt/mgmt_snode/src/smHandle.c   |   1 +
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c   |   2 +
 source/dnode/snode/inc/sndInt.h               |  17 --
 source/dnode/snode/src/snode.c                |   2 +
 source/dnode/snode/src/snodeInitApi.c         |   6 +-
 source/dnode/vnode/src/inc/vnodeInt.h         |   1 +
 source/dnode/vnode/src/tq/tq.c                |  16 +-
 source/dnode/vnode/src/tqCommon/tqCommon.c    |  43 +++--
 source/dnode/vnode/src/vnd/vnodeSvr.c         |   5 +
 source/libs/stream/inc/streamBackendRocksdb.h |   2 +-
 source/libs/stream/inc/streamInt.h            |   3 +-
 source/libs/stream/src/streamBackendRocksdb.c | 150 +++++++++---------
 source/libs/stream/src/streamCheckpoint.c     | 144 +++++++++--------
 source/libs/stream/src/streamDispatch.c       |   4 +-
 source/libs/stream/src/streamExec.c           |   2 +-
 source/libs/stream/src/streamMeta.c           |   9 +-
 source/libs/stream/src/streamTask.c           |  37 ++++-
 22 files changed, 272 insertions(+), 206 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index ed23290be4..790b8f883b 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3440,23 +3440,24 @@ int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamR
 int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq);
 void    tFreeMDropStreamReq(SMDropStreamReq* pReq);
 
-typedef struct {
-  char   name[TSDB_STREAM_FNAME_LEN];
-  int8_t igNotExists;
-} SMRecoverStreamReq;
-
-typedef struct {
-  int8_t reserved;
-} SMRecoverStreamRsp;
-
 typedef struct {
   int64_t recoverObjUid;
   int32_t taskId;
   int32_t hasCheckPoint;
 } SMVStreamGatherInfoReq;
 
-// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq);
-// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq);
+typedef struct SVUpdateCheckpointInfoReq {
+  SMsgHead head;
+  int64_t  streamId;
+  int32_t  taskId;
+  int64_t  checkpointId;
+  int64_t  checkpointVer;
+  int64_t  checkpointTs;
+  int32_t  transId;
+  int8_t   dropRelHTask;
+  int64_t  hStreamId;
+  int64_t  hTaskId;
+} SVUpdateCheckpointInfoReq;
 
 typedef struct {
   int64_t leftForVer;
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index a5a3bd5ee0..12f92b1242 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -310,7 +310,7 @@
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_UPDATE_CHKPT, "stream-update-chkptinfo", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)  //1035 1036
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT_READY, "stream-checkpoint-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL)
@@ -321,6 +321,7 @@
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL)
+
   TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
 
   TD_NEW_MSG_SEG(TDMT_MON_MSG)  //5 << 8
diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h
index ce04ec6953..451f9a00eb 100644
--- a/include/dnode/vnode/tqCommon.h
+++ b/include/dnode/vnode/tqCommon.h
@@ -38,6 +38,7 @@ int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta);
 int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
 int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
+int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
 
 int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode);
 void    tqSetRestoreVersionInfo(SStreamTask* pTask);
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 3c5d6d6e4c..9d5b7bc6f1 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -736,6 +736,9 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask);
 int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,
                                            int32_t setCode);
+int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId,
+                                              SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
+int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq);
 
 // stream task state machine, and event handling
 SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index e448aec5e0..867b407296 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -208,11 +208,11 @@ int32_t downloadRsync(const char* id, const char* path) {
 
   int32_t code = execCommand(command);
   if (code != 0) {
-    uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    uError("[rsync] download checkpoint data failed, code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
     return -1;
   }
 
-  uDebug("[rsync] down data:%s successful", id);
+  uDebug("[rsync] download checkpoint data:%s successfully", id);
   return 0;
 }
 
diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
index 880e96adfb..b3c8ef4017 100644
--- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c
+++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
@@ -78,6 +78,7 @@ SArray *smGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 73a73d19f5..826af71ace 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -967,6 +967,8 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
 
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h
index 8c5d056893..2ac66fa1cd 100644
--- a/source/dnode/snode/inc/sndInt.h
+++ b/source/dnode/snode/inc/sndInt.h
@@ -35,23 +35,6 @@ struct SSnode {
   SMsgCb       msgCb;
 };
 
-#if 0
-typedef struct {
-  SHashObj* pHash;  // taskId -> SStreamTask
-} SStreamMeta;
-
-SStreamMeta* sndMetaNew();
-void         sndMetaDelete(SStreamMeta* pMeta);
-
-int32_t      sndMetaDeployTask(SStreamMeta* pMeta, SStreamTask* pTask);
-SStreamTask* sndMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
-int32_t      sndMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
-
-int32_t sndDropTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
-int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
-int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
-#endif
-
 void initStreamStateAPI(SStorageAPI* pAPI);
 
 #ifdef __cplusplus
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 87f0681780..602264be73 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -168,6 +168,8 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
       return tqStreamTaskProcessTaskPauseReq(pSnode->pMeta, pMsg->pCont);
     case TDMT_STREAM_TASK_RESUME:
       return tqStreamTaskProcessTaskResumeReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pMsg->pCont, false);
+    case TDMT_STREAM_TASK_UPDATE_CHKPT:
+      return tqStreamTaskProcessUpdateCheckpointReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen);
     default:
       ASSERT(0);
   }
diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c
index 3b60ef3427..196fa56c99 100644
--- a/source/dnode/snode/src/snodeInitApi.c
+++ b/source/dnode/snode/src/snodeInitApi.c
@@ -75,8 +75,10 @@ void initStateStoreAPI(SStateStore* pStore) {
   pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange;
   pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition;
 
-  pStore->streamStateCountWinAddIfNotExist = streamStateCountWinAddIfNotExist;
-  pStore->streamStateCountWinAdd = streamStateCountWinAdd;
+//void initStreamStateAPI(SStorageAPI* pAPI) {
+//  initStateStoreAPI(&pAPI->stateStore);
+//  initFunctionStateStore(&pAPI->functionStore);
+//}
 
   pStore->updateInfoInit = updateInfoInit;
   pStore->updateInfoFillBlockData = updateInfoFillBlockData;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 419ebd1a6c..9439f7f179 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -295,6 +295,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqStreamProgressRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen);
 
 // sma
 int32_t smaInit();
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 79f53e6dec..b75d517997 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -828,7 +828,7 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask
 
     streamSetParamForStreamScannerStep2(pTask, pStep2Range, pWindow);
 
-    int64_t dstVer =pStep2Range->minVer;
+    int64_t dstVer = pStep2Range->minVer;
     pTask->chkInfo.nextProcessVer = dstVer;
 
     walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer);
@@ -1009,6 +1009,20 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
   return tqStreamTaskProcessDropReq(pTq->pStreamMeta, msg, msgLen);
 }
 
+int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) {
+  int32_t vgId = TD_VID(pTq->pVnode);
+  SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
+
+  if (!pTq->pVnode->restored) {
+    tqDebug("vgId:%d update-checkpoint-info msg received during restoring, checkpointId:%" PRId64
+            ", transId:%d s-task:0x%x ignore it",
+            vgId, pReq->checkpointId, pReq->transId, pReq->taskId);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, msg, msgLen);
+}
+
 int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
   return tqStreamTaskProcessTaskPauseReq(pTq->pStreamMeta, msg);
 }
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 62c3b06b65..91de290e6a 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -65,6 +65,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode)
       .fillHistory = pTask->info.fillHistory,
       .winRange = pTask->dataRange.window,
   };
+
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     handle.vnode = pVnode;
     handle.initTqReader = 1;
@@ -577,22 +578,23 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
   return code;
 }
 
-void tqStreamRmTaskBackend(SStreamMeta* pMeta, STaskId* id) {
+static void tqStreamRemoveTaskBackend(SStreamMeta* pMeta, const STaskId* pId) {
   char taskKey[128] = {0};
-  sprintf(taskKey, "0x%" PRIx64 "-0x%x", id->streamId, (int32_t)id->taskId);
+  sprintf(taskKey, "0x%" PRIx64 "-0x%x", pId->streamId, (int32_t)pId->taskId);
 
   char* path = taosMemoryCalloc(1, strlen(pMeta->path) + 128);
   sprintf(path, "%s%s%s", pMeta->path, TD_DIRSEP, taskKey);
   taosRemoveDir(path);
+
+  tqInfo("vgId:%d drop stream task:0x%x file:%s", pMeta->vgId, (int32_t)pId->taskId, path);
   taosMemoryFree(path);
-  // do nothing
 }
 
 int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
   SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
-  int32_t              vgId = pMeta->vgId;
-  STaskId              hTaskId = {0};
 
+  int32_t vgId = pMeta->vgId;
+  STaskId hTaskId = {0};
   tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
 
   streamMetaWLock(pMeta);
@@ -634,10 +636,32 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
 
   streamMetaWUnLock(pMeta);
 
-  tqStreamRmTaskBackend(pMeta, &id);
+  tqStreamRemoveTaskBackend(pMeta, &id);
   return 0;
 }
 
+int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
+  SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
+
+  int32_t vgId = pMeta->vgId;
+  tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId);
+
+  streamMetaWLock(pMeta);
+
+  STaskId       id = {.streamId = pReq->streamId, .taskId = pReq->taskId};
+  SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
+
+  if (ppTask != NULL && (*ppTask) != NULL) {
+    streamTaskUpdateTaskCheckpointInfo(*ppTask, pReq);
+  } else {  // failed to get the task.
+    tqError("vgId:%d failed to locate the s-task:0x%x to update the checkpoint info, it may have been dropped already",
+            vgId, pReq->taskId);
+  }
+
+  streamMetaWUnLock(pMeta);
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
   int32_t vgId = pMeta->vgId;
   int32_t code = 0;
@@ -927,9 +951,10 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
 
 int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) {
   SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg;
-  SStreamMeta*           pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
-  SStreamTask*           pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
-  int32_t                code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode);
+
+  SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
+  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
+  int32_t      code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode);
   if (code != 0) {
     return code;
   }
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 6d97c1cd79..3fec3d9b8e 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -625,6 +625,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
         goto _err;
       }
     } break;
+    case TDMT_STREAM_TASK_UPDATE_CHKPT: {
+      if (tqProcessTaskUpdateCheckpointReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) {
+        goto _err;
+      }
+    } break;
     case TDMT_STREAM_TASK_PAUSE: {
       if (pVnode->restored && vnodeIsLeader(pVnode) &&
           tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index 704bc9a2f2..fbf902a237 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -141,7 +141,7 @@ SListNode* streamBackendAddCompare(void* backend, void* arg);
 void       streamBackendDelCompare(void* backend, void* arg);
 int32_t    streamStateCvtDataFormat(char* path, char* key, void* cfInst);
 
-STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId);
+STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkptId);
 void            taskDbDestroy(void* pBackend, bool flush);
 void            taskDbDestroy2(void* pBackend);
 int32_t         taskDbDoCheckpoint(void* arg, int64_t chkpId);
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 3ccb25a62a..da8a24e6da 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -113,7 +113,7 @@ void    destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups);
 int32_t getNumOfDispatchBranch(SStreamTask* pTask);
 void    clearBufferedDispatchMsg(SStreamTask* pTask);
 
-int32_t           streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
+int32_t           streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
 SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg);
 SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize,
                                                SArray* pRes);
@@ -122,7 +122,6 @@ void              destroyStreamDataBlock(SStreamDataBlock* pBlock);
 int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData);
 int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock);
 
-int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId);
 int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet);
 
 int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId);
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index b5294a3fb7..d9eea23d21 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -42,7 +42,7 @@ void            destroyRocksdbCfInst(RocksdbCfInst* inst);
 int32_t         getCfIdx(const char* cfName);
 STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath);
 
-int32_t backendCopyFiles(char* src, char* dst);
+static int32_t backendCopyFiles(const char* src, const char* dst);
 
 void        destroyCompactFilteFactory(void* arg);
 void        destroyCompactFilte(void* arg);
@@ -234,12 +234,14 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
               tstrerror(TAOS_SYSTEM_ERROR(errno)), state);
       taosMkDir(state);
     }
+
     taosMemoryFree(chkp);
   }
-  *dst = state;
 
+  *dst = state;
   return 0;
 }
+
 int32_t remoteChkp_readMetaData(char* path, SArray* list) {
   char* metaPath = taosMemoryCalloc(1, strlen(path));
   sprintf(metaPath, "%s%s%s", path, TD_DIRSEP, "META");
@@ -323,7 +325,7 @@ int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) {
   return complete == 1 ? 0 : -1;
 }
 
-int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
+int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkpPath, int64_t checkpointId, char* defaultPath) {
   // impl later
   int32_t code = 0;
   if (taosIsDir(chkpPath)) {
@@ -336,10 +338,14 @@ int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkpPath, int64_t chkpId, c
 
   code = streamTaskDownloadCheckpointData(key, chkpPath);
   if (code != 0) {
+    stError("failed to download checkpoint data:%s", key);
     return code;
   }
 
+  stDebug("download backup checkpoint data into:%s, checkpointId:%" PRId64 ", %s", chkpPath, checkpointId, key);
+
   code = backendCopyFiles(chkpPath, defaultPath);
+
   return code;
 }
 
@@ -378,13 +384,16 @@ int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char
   return code;
 }
 
-int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
+int32_t rebuildFromRemoteCheckpoint(char* key, char* chkpPath, int64_t checkpointId, char* defaultPath) {
   ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
   if (type == DATA_UPLOAD_S3) {
-    return rebuildFromRemoteChkp_s3(key, chkpPath, chkpId, defaultPath);
+    return rebuildFromRemoteChkp_s3(key, chkpPath, checkpointId, defaultPath);
   } else if (type == DATA_UPLOAD_RSYNC) {
-    return rebuildFromRemoteChkp_rsync(key, chkpPath, chkpId, defaultPath);
+    return rebuildFromRemoteChkp_rsync(key, chkpPath, checkpointId, defaultPath);
+  } else {
+    stError("%s not remote backup checkpoint data for:%"PRId64, key, checkpointId);
   }
+
   return -1;
 }
 
@@ -403,7 +412,7 @@ int32_t copyFiles_hardlink(char* src, char* dst, int8_t type) {
   return taosLinkFile(src, dst);
 }
 
-int32_t backendFileCopyFilesImpl(char* src, char* dst) {
+int32_t backendFileCopyFilesImpl(const char* src, const char* dst) {
   const char* current = "CURRENT";
   size_t      currLen = strlen(current);
 
@@ -412,20 +421,26 @@ int32_t backendFileCopyFilesImpl(char* src, char* dst) {
   int32_t dLen = strlen(dst);
   char*   srcName = taosMemoryCalloc(1, sLen + 64);
   char*   dstName = taosMemoryCalloc(1, dLen + 64);
-  // copy file to dst
 
+  // copy file to dst
   TdDirPtr pDir = taosOpenDir(src);
   if (pDir == NULL) {
     taosMemoryFree(srcName);
     taosMemoryFree(dstName);
+    code = TAOS_SYSTEM_ERROR(errno);
+
     errno = 0;
-    return -1;
+    return code;
   }
 
+  errno = 0;
   TdDirEntryPtr de = NULL;
+
   while ((de = taosReadDir(pDir)) != NULL) {
     char* name = taosGetDirEntryName(de);
-    if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
+    if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) {
+      continue;
+    }
 
     sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name);
     sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name);
@@ -433,18 +448,21 @@ int32_t backendFileCopyFilesImpl(char* src, char* dst) {
     if (strncmp(name, current, strlen(name) <= currLen ? strlen(name) : currLen) == 0) {
       code = copyFiles_create(srcName, dstName, 0);
       if (code != 0) {
-        stError("failed to copy file, detail: %s to %s reason: %s", srcName, dstName,
-                tstrerror(TAOS_SYSTEM_ERROR(code)));
+        code = TAOS_SYSTEM_ERROR(code);
+        stError("failed to copy file, detail: %s to %s reason:%s", srcName, dstName, tstrerror(code));
         goto _ERROR;
       }
     } else {
       code = copyFiles_hardlink(srcName, dstName, 0);
       if (code != 0) {
-        stError("failed to hard line file, detail: %s to %s, reason: %s", srcName, dstName,
-                tstrerror(TAOS_SYSTEM_ERROR(code)));
+        code = TAOS_SYSTEM_ERROR(code);
+        stError("failed to hardlink file, detail:%s to %s, reason:%s", srcName, dstName, tstrerror(code));
         goto _ERROR;
+      } else {
+        stDebug("succ hard link file:%s to %s", srcName, dstName);
       }
     }
+
     memset(srcName, 0, sLen + 64);
     memset(dstName, 0, dLen + 64);
   }
@@ -453,76 +471,46 @@ int32_t backendFileCopyFilesImpl(char* src, char* dst) {
   taosMemoryFreeClear(dstName);
   taosCloseDir(&pDir);
   errno = 0;
-  return 0;
+  return code;
+
 _ERROR:
   taosMemoryFreeClear(srcName);
   taosMemoryFreeClear(dstName);
   taosCloseDir(&pDir);
   errno = 0;
-  return -1;
+  return code;
 }
-int32_t backendCopyFiles(char* src, char* dst) {
+
+int32_t backendCopyFiles(const char* src, const char* dst) {
   return backendFileCopyFilesImpl(src, dst);
-  // // opt later, just hard link
-  // int32_t sLen = strlen(src);
-  // int32_t dLen = strlen(dst);
-  // char*   srcName = taosMemoryCalloc(1, sLen + 64);
-  // char*   dstName = taosMemoryCalloc(1, dLen + 64);
-
-  // TdDirPtr pDir = taosOpenDir(src);
-  // if (pDir == NULL) {
-  //   taosMemoryFree(srcName);
-  //   taosMemoryFree(dstName);
-  //   return -1;
-  // }
-
-  // TdDirEntryPtr de = NULL;
-  // while ((de = taosReadDir(pDir)) != NULL) {
-  //   char* name = taosGetDirEntryName(de);
-  //   if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
-
-  //   sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name);
-  //   sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name);
-  //   // if (!taosDirEntryIsDir(de)) {
-  //   //   // code = taosCopyFile(srcName, dstName);
-  //   //   if (code == -1) {
-  //   //     goto _err;
-  //   //   }
-  //   // }
-  //   return backendFileCopyFilesImpl(src, dst);
-
-  //   memset(srcName, 0, sLen + 64);
-  //   memset(dstName, 0, dLen + 64);
-  // }
-
-  // _err:
-  //   taosMemoryFreeClear(srcName);
-  //   taosMemoryFreeClear(dstName);
-  //   taosCloseDir(&pDir);
-  //   return code >= 0 ? 0 : -1;
-
-  // return 0;
 }
-int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
+
+static int32_t rebuildFromLocalCheckpoint(char* pTaskIdStr, const char* checkpointPath, int64_t chkpId, const char* defaultPath) {
   int32_t code = 0;
+
   if (taosIsDir(defaultPath)) {
     taosRemoveDir(defaultPath);
     taosMkDir(defaultPath);
 
-    stInfo("succ to clear stream backend %s", defaultPath);
+    stInfo("clear task backend path:%s, done", defaultPath);
   }
-  if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) {
-    code = backendCopyFiles(chkpPath, defaultPath);
-    if (code != 0) {
+
+  if (taosIsDir(checkpointPath) && isValidCheckpoint(checkpointPath)) {
+    code = backendCopyFiles(checkpointPath, defaultPath);
+
+    if (code != TSDB_CODE_SUCCESS) {
       taosRemoveDir(defaultPath);
       taosMkDir(defaultPath);
 
-      stError("failed to restart stream backend from %s, reason: %s, start to restart from empty path: %s", chkpPath,
-              tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath);
-      code = 0;
+      stError("%s failed to restart stream backend from %s, reason: %s, start to restart from empty path: %s",
+              pTaskIdStr, checkpointPath, tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath);
+      code = TSDB_CODE_SUCCESS;
     } else {
-      stInfo("start to restart stream backend at checkpoint path: %s", chkpPath);
+      stInfo("%s start to restart stream backend at checkpoint path: %s", pTaskIdStr, checkpointPath);
     }
+  } else {
+    code = TSDB_CODE_FAILED;
+    stError("%s not valid checkpoint path/data in:%s", pTaskIdStr, checkpointPath);
   }
 
   return code;
@@ -533,7 +521,7 @@ int32_t rebuildFromlocalDefault(char* key, char* chkpPath, int64_t chkpId, char*
   return code;
 }
 
-int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** dbPrefixPath, char** dbPath) {
+int32_t rebuildDirFormCheckpoint(const char* path, char* key, int64_t chkptId, char** dbPrefixPath, char** dbPath) {
   // impl later
   int32_t code = 0;
 
@@ -551,29 +539,33 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char**
     taosMulMkDir(defaultPath);
   }
 
-  char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256);
-  if (chkpId != 0) {
-    sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId);
-    code = rebuildFromLocalChkp(key, chkpPath, chkpId, defaultPath);
+  stDebug("prepare local dir:%s, checkpointId:%d, key:%s succ", defaultPath, chkptId, key);
+
+  char* chkptPath = taosMemoryCalloc(1, strlen(path) + 256);
+  if (chkptId != 0) {
+    sprintf(chkptPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId);
+
+    code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath);
     if (code != 0) {
-      code = rebuildFromRemoteChkp(key, chkpPath, chkpId, defaultPath);
+      code = rebuildFromRemoteCheckpoint(key, chkptPath, chkptId, defaultPath);
     }
 
     if (code != 0) {
-      stInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath,
-             tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath);
+      stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath,
+             tstrerror(code), defaultPath);
       code = taosMkDir(defaultPath);
     }
   } else {
-    sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint",
+    sprintf(chkptPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint",
             (int64_t)-1);
 
-    code = rebuildFromLocalChkp(key, chkpPath, -1, defaultPath);
+    code = rebuildFromLocalCheckpoint(key, chkptPath, -1, defaultPath);
     if (code != 0) {
       code = taosMkDir(defaultPath);
     }
   }
-  taosMemoryFree(chkpPath);
+
+  taosMemoryFree(chkptPath);
 
   *dbPath = defaultPath;
   *dbPrefixPath = prefixPath;
@@ -1055,6 +1047,7 @@ _ERROR:
   rocksdb_checkpoint_object_destroy(cp);
   return code;
 }
+
 int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32_t nCf) {
   if (nCf == 0) return 0;
   int   code = 0;
@@ -1098,6 +1091,7 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI
 
   return 0;
 }
+
 int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
   SStreamMeta* pMeta = arg;
 
@@ -1997,11 +1991,11 @@ _EXIT:
   return NULL;
 }
 
-STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) {
+STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkptId) {
   char* statePath = NULL;
   char* dbPath = NULL;
 
-  if (rebuildDirFromChkp2(path, key, chkpId, &statePath, &dbPath) != 0) {
+  if (rebuildDirFormCheckpoint(path, key, chkptId, &statePath, &dbPath) != 0) {
     return NULL;
   }
 
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 5a4e3a5439..853c6881ba 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -33,8 +33,11 @@ static int32_t deleteCheckpointFile(const char* id, const char* name);
 static int32_t streamTaskBackupCheckpoint(const char* id, const char* path);
 static int32_t deleteCheckpoint(const char* id);
 static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName);
+static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask);
+static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType);
+static int32_t streamAlignCheckpoint(SStreamTask* pTask);
 
-static int32_t streamAlignCheckpoint(SStreamTask* pTask) {
+int32_t streamAlignCheckpoint(SStreamTask* pTask) {
   int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
   int64_t old = atomic_val_compare_exchange_32(&pTask->chkInfo.downstreamAlignNum, 0, num);
   if (old == 0) {
@@ -44,7 +47,7 @@ static int32_t streamAlignCheckpoint(SStreamTask* pTask) {
   return atomic_sub_fetch_32(&pTask->chkInfo.downstreamAlignNum, 1);
 }
 
-static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) {
+int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) {
   SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock));
   if (pChkpoint == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -94,7 +97,7 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo
   return appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER);
 }
 
-static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) {
+int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) {
   pBlock->srcTaskId = pTask->id.taskId;
   pBlock->srcVgId = pTask->pMeta->vgId;
 
@@ -110,7 +113,7 @@ static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStream
   return code;
 }
 
-int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) {
+int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) {
   SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0);
   int64_t      checkpointId = pDataBlock->info.version;
   int32_t      transId = pDataBlock->info.window.skey;
@@ -118,7 +121,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
   int32_t      code = TSDB_CODE_SUCCESS;
   int32_t      vgId = pTask->pMeta->vgId;
 
-  stDebug("s-task:%s vgId:%d start to handle the checkpoint block, checkpointId:%" PRId64 " ver:%" PRId64
+  stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64
           ", transId:%d current checkpointingId:%" PRId64,
           id, vgId, pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, transId, checkpointId);
 
@@ -141,7 +144,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
     int8_t type = pTask->outputInfo.type;
     if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
       stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
-      continueDispatchCheckpointBlock(pBlock, pTask);
+      continueDispatchCheckpointTriggerBlock(pBlock, pTask);
     } else {  // only one task exists, no need to dispatch downstream info
       atomic_add_fetch_32(&pTask->chkInfo.numOfNotReady, 1);
       streamProcessCheckpointReadyMsg(pTask);
@@ -182,7 +185,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
 
       // Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task
       // already. And then, dispatch check point msg to all downstream tasks
-      code = continueDispatchCheckpointBlock(pBlock, pTask);
+      code = continueDispatchCheckpointTriggerBlock(pBlock, pTask);
     }
   }
 
@@ -227,75 +230,83 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
   }
 }
 
-int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) {
-  SStreamMeta*     pMeta = p->pMeta;
+int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq) {
+  SStreamMeta*     pMeta = pTask->pMeta;
   int32_t          vgId = pMeta->vgId;
-  const char*      id = p->id.idStr;
   int32_t          code = 0;
-  SCheckpointInfo* pCKInfo = &p->chkInfo;
+  const char*      id = pTask->id.idStr;
+  SCheckpointInfo* pInfo = &pTask->chkInfo;
 
-  // fill-history task, rsma task, and sink task will not generate the checkpoint
-  if ((p->info.fillHistory == 1) || (p->info.taskLevel > TASK_LEVEL__SINK)) {
-    return code;
-  }
+  taosThreadMutexLock(&pTask->lock);
 
-  taosThreadMutexLock(&p->lock);
+  SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
 
-  SStreamTaskState* pStatus = streamTaskGetStatus(p);
-  ETaskStatus       prevStatus = pStatus->state;
+  stDebug("s-task:%s vgId:%d status:%s start to update the checkpoint info, checkpointId:%" PRId64 "->%" PRId64
+          " checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64,
+          id, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer,
+          pInfo->checkpointTime, pReq->checkpointTs);
 
-  if (pStatus->state == TASK_STATUS__CK) {
-    ASSERT(pCKInfo->checkpointId <= pCKInfo->checkpointingId && pCKInfo->checkpointingId == checkpointId &&
-           pCKInfo->checkpointVer <= pCKInfo->processedVer);
+  // in the
+  if (pStatus->state != TASK_STATUS__DROPPING) {
+    ASSERT(pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer);
 
-    pCKInfo->checkpointId = pCKInfo->checkpointingId;
-    pCKInfo->checkpointVer = pCKInfo->processedVer;
-    pCKInfo->checkpointTime = pCKInfo->startTs;
+    pInfo->checkpointId = pReq->checkpointId;
+    pInfo->checkpointVer = pReq->checkpointVer;
+    pInfo->checkpointTime = pReq->checkpointTs;
 
-    streamTaskClearCheckInfo(p, false);
-    taosThreadMutexUnlock(&p->lock);
+    streamTaskClearCheckInfo(pTask, false);
 
-    code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
+    // todo handle error
+    if (pStatus->state == TASK_STATUS__CK) {
+      code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
+    } else {
+      stDebug("s-task:0x%x vgId:%d not handle checkpoint-done event, status:%s", pReq->taskId, vgId, pStatus->name);
+    }
   } else {
-    stDebug("s-task:%s vgId:%d status:%s not keep the checkpoint metaInfo, checkpoint:%" PRId64 " failed", id, vgId,
-            pStatus->name, pCKInfo->checkpointingId);
-    taosThreadMutexUnlock(&p->lock);
+    stDebug("s-task:0x%x vgId:%d status:%s not update checkpoint info, checkpointId:%" PRId64 "->%" PRId64 " failed",
+            pReq->taskId, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId);
+    taosThreadMutexUnlock(&pTask->lock);
 
     return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
   }
 
+  if (pReq->dropRelHTask) {
+    stDebug("s-task:0x%x vgId:%d drop the related fill-history task:0x%" PRIx64 " after update checkpoint",
+            pReq->taskId, vgId, pReq->hTaskId);
+    CLEAR_RELATED_FILLHISTORY_TASK(pTask);
+  }
+
+  stDebug("s-task:0x%x set the persistent status attr to be ready, prev:%s, status in sm:%s", pReq->taskId,
+          streamTaskGetStatusStr(pTask->status.taskStatus), streamTaskGetStatus(pTask)->name);
+
+  pTask->status.taskStatus = TASK_STATUS__READY;
+
+  code = streamMetaSaveTask(pMeta, pTask);
   if (code != TSDB_CODE_SUCCESS) {
-    stDebug("s-task:%s vgId:%d handle event:checkpoint-done failed", id, vgId);
+    stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id,
+            vgId, pReq->checkpointId, terrstr());
     return code;
   }
 
-  stDebug("vgId:%d s-task:%s level:%d open upstream inputQ, save status after checkpoint, checkpointId:%" PRId64
-          ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status: ready, prev:%s",
-          vgId, id, p->info.taskLevel, checkpointId, pCKInfo->checkpointVer, pCKInfo->nextProcessVer,
-          streamTaskGetStatusStr(prevStatus));
+  taosThreadMutexUnlock(&pTask->lock);
+  streamMetaWUnLock(pMeta);
 
-  // save the task if not sink task
-  if (p->info.taskLevel <= TASK_LEVEL__SINK) {
-    streamMetaWLock(pMeta);
+  // drop task should not in the meta-lock, and drop the related fill-history task now
+  if (pReq->dropRelHTask) {
+    streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
 
-    code = streamMetaSaveTask(pMeta, p);
-    if (code != TSDB_CODE_SUCCESS) {
-      streamMetaWUnLock(pMeta);
-      stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id,
-              vgId, checkpointId, terrstr());
-      return code;
-    }
-
-    code = streamMetaCommit(pMeta);
-    if (code != TSDB_CODE_SUCCESS) {
-      stError("s-task:%s vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s",
-              id, vgId, checkpointId, terrstr());
-    }
-
-    streamMetaWUnLock(pMeta);
+    // commit the update
+    int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
+    stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, pReq->taskId, numOfTasks);
   }
 
-  return code;
+  streamMetaWLock(pMeta);
+
+  if (streamMetaCommit(pMeta) < 0) {
+    // persist to disk
+  }
+
+  return TSDB_CODE_SUCCESS;
 }
 
 void streamTaskSetFailedCheckpointId(SStreamTask* pTask) {
@@ -334,6 +345,7 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l
       }
     }
   }
+
   taosCloseFile(&pFile);
   taosRemoveFile(file);
   taosMemoryFree(file);
@@ -448,7 +460,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
 
   // clear the checkpoint info, and commit the newest checkpoint info if all works are done successfully
   if (code == TSDB_CODE_SUCCESS) {
-    code = streamSaveTaskCheckpointInfo(pTask, ckId);
+    STaskId* pHTaskId = &pTask->hTaskInfo.id;
+    code = streamBuildAndSendCheckpointUpdateMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, pHTaskId, &pTask->chkInfo,
+                                                 dropRelHTask);
     if (code == TSDB_CODE_SUCCESS) {
       code = streamTaskRemoteBackupCheckpoint(pTask, ckId, (char*)id);
       if (code != TSDB_CODE_SUCCESS) {
@@ -459,22 +473,6 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
     }
   }
 
-  if ((code == TSDB_CODE_SUCCESS) && dropRelHTask) {
-    // transferred from the halt status, it is done the fill-history procedure and finish with the checkpoint
-    // free it and remove fill-history task from disk meta-store
-    taosThreadMutexLock(&pTask->lock);
-    if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
-      SStreamTaskId hTaskId = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId};
-
-      stDebug("s-task:%s fill-history finish checkpoint done, drop related fill-history task:0x%x", id, hTaskId.taskId);
-      streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &hTaskId, 1);
-    } else {
-      stWarn("s-task:%s related fill-history task:0x%x is erased", id, (int32_t)pTask->hTaskInfo.id.taskId);
-    }
-
-    taosThreadMutexUnlock(&pTask->lock);
-  }
-
   // clear the checkpoint info if failed
   if (code != TSDB_CODE_SUCCESS) {
     taosThreadMutexLock(&pTask->lock);
@@ -488,7 +486,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
 
   double el = (taosGetTimestampMs() - startTs) / 1000.0;
   stInfo("s-task:%s vgId:%d level:%d, checkpointId:%" PRId64 " ver:%" PRId64 " elapsed time:%.2f Sec, %s ", id,
-         pTask->pMeta->vgId, pTask->info.taskLevel, ckId, pTask->chkInfo.checkpointVer, el,
+         pMeta->vgId, pTask->info.taskLevel, ckId, pTask->chkInfo.checkpointVer, el,
          (code == TSDB_CODE_SUCCESS) ? "succ" : "failed");
 
   return code;
@@ -586,7 +584,7 @@ int32_t downloadCheckpointDataByName(const char* id, const char* fname, const ch
 
 int32_t streamTaskDownloadCheckpointData(char* id, char* path) {
   if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) {
-    stError("streamTaskDownloadCheckpointData parameters invalid");
+    stError("down checkpoint data parameters invalid");
     return -1;
   }
 
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 58c6e19581..f3569d8973 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -341,7 +341,7 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch
     SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet;
     int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
 
-    stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), id:%d", id,
+    stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d", id,
             pTask->info.selfChildId, 1, downstreamTaskId, vgId, msgId);
 
     code = doSendDispatchMsg(pTask, pDispatchMsg, vgId, pEpSet);
@@ -432,7 +432,7 @@ static void doRetryDispatchData(void* param, void* tmrId) {
       SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet;
       int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
 
-      stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), id:%d", id,
+      stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d", id,
               pTask->info.selfChildId, 1, downstreamTaskId, vgId, msgId);
 
       code = doSendDispatchMsg(pTask, pReq, vgId, pEpSet);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 047b169ec9..9747ebd2ff 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -589,7 +589,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
     // dispatch checkpoint msg to all downstream tasks
     int32_t type = pInput->type;
     if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
-      streamProcessCheckpointBlock(pTask, (SStreamDataBlock*)pInput);
+      streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
       continue;
     }
 
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 4fa9b2c66f..95fd057929 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -556,19 +556,22 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){
     pTask->ver = SSTREAM_TASK_VER;
   }
+
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, len);
   tEncodeStreamTask(&encoder, pTask);
   tEncoderClear(&encoder);
 
   int64_t id[2] = {pTask->id.streamId, pTask->id.taskId};
-  if (tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn) < 0) {
+  code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn);
+  if (code < 0) {
     stError("s-task:%s save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno));
-    return -1;
+  } else {
+    stDebug("s-task:%s vgId:%d stream task write to meta file", pTask->id.idStr, pTask->pMeta->vgId);
   }
 
   taosMemoryFree(buf);
-  return 0;
+  return code;
 }
 
 int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pTaskId) {
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 1e622f615d..72302f981d 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -686,16 +686,47 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
   pReq->head.vgId = vgId;
   pReq->taskId = pTaskId->taskId;
   pReq->streamId = pTaskId->streamId;
-  pReq->resetRelHalt = resetRelHalt;
+  pReq->resetRelHalt = resetRelHalt;  // todo: remove this attribute
 
   SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_DROP, .pCont = pReq, .contLen = sizeof(SVDropStreamTaskReq)};
   int32_t code = tmsgPutToQueue(pMsgCb, WRITE_QUEUE, &msg);
   if (code != TSDB_CODE_SUCCESS) {
     stError("vgId:%d failed to send drop task:0x%x msg, code:%s", vgId, pTaskId->taskId, tstrerror(code));
-    return code;
+  } else {
+    stDebug("vgId:%d build and send drop task:0x%x msg", vgId, pTaskId->taskId);
   }
 
-  stDebug("vgId:%d build and send drop task:0x%x msg", vgId, pTaskId->taskId);
+  return code;
+}
+
+int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId,
+                                              SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) {
+  SVUpdateCheckpointInfoReq* pReq = rpcMallocCont(sizeof(SVUpdateCheckpointInfoReq));
+  if (pReq == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pReq->head.vgId = vgId;
+  pReq->taskId = pTaskId->taskId;
+  pReq->streamId = pTaskId->streamId;
+  pReq->dropRelHTask = dropRelHTask;
+  pReq->hStreamId = pHTaskId->streamId;
+  pReq->hTaskId = pHTaskId->taskId;
+  pReq->transId = pCheckpointInfo->transId;
+
+  pReq->checkpointId = pCheckpointInfo->checkpointingId;
+  pReq->checkpointVer = pCheckpointInfo->processedVer;
+  pReq->checkpointTs = pCheckpointInfo->startTs;
+
+  SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_UPDATE_CHKPT, .pCont = pReq, .contLen = sizeof(SVUpdateCheckpointInfoReq)};
+  int32_t code = tmsgPutToQueue(pMsgCb, WRITE_QUEUE, &msg);
+
+  if (code != TSDB_CODE_SUCCESS) {
+    stError("vgId:%d task:0x%x failed to send update checkpoint info msg, code:%s", vgId, pTaskId->taskId, tstrerror(code));
+  } else {
+    stDebug("vgId:%d task:0x%x build and send update checkpoint info msg msg", vgId, pTaskId->taskId);
+  }
   return code;
 }
 

From 11ed1f54b3f37b1d74705699ad1e47ff5f3543d0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 11 May 2024 10:09:23 +0800
Subject: [PATCH 002/132] refactor: do some internal refactor.

---
 source/libs/stream/src/streamBackendRocksdb.c |  1 +
 source/libs/stream/src/streamCheckpoint.c     | 57 ++++++++++++-------
 2 files changed, 36 insertions(+), 22 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index d9eea23d21..56ea1611f4 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -1209,6 +1209,7 @@ _EXIT:
   taosMemoryFree(ppCf);
   return code;
 }
+
 int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskDbDoCheckpoint(arg, chkpId); }
 
 SListNode* streamBackendAddCompare(void* backend, void* arg) {
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 853c6881ba..a1e6838ac3 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -353,42 +353,51 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l
 }
 
 int32_t uploadCheckpointData(void* param) {
-  SAsyncUploadArg* arg = param;
+  SAsyncUploadArg* pParam = param;
   char*            path = NULL;
   int32_t          code = 0;
   SArray*          toDelFiles = taosArrayInit(4, sizeof(void*));
-  char*            taskStr = arg->taskId ? arg->taskId : "NULL";
+  char*            taskStr = pParam->taskId ? pParam->taskId : "NULL";
 
-  void* pBackend = taskAcquireDb(arg->dbRefId);
+  void* pBackend = taskAcquireDb(pParam->dbRefId);
   if (pBackend == NULL) {
     stError("s-task:%s failed to acquire db", taskStr);
-    taosMemoryFree(arg->taskId);
-    taosMemoryFree(arg);
+    taosMemoryFree(pParam->taskId);
+    taosMemoryFree(pParam);
     return -1;
   }
 
-  if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, ((SStreamMeta*)arg->pMeta)->bkdChkptMgt, arg->chkpId,
-                                      (int8_t)(arg->type), &path, toDelFiles)) != 0) {
-    stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", taskStr, arg->chkpId);
+  if ((code = taskDbGenChkpUploadData(pParam->pTask->pBackend, ((SStreamMeta*)pParam->pMeta)->bkdChkptMgt,
+                                      pParam->chkpId,
+                                      (int8_t)(pParam->type), &path, toDelFiles)) != 0) {
+    stError("s-task:%s failed to gen upload checkpoint:%" PRId64, taskStr, pParam->chkpId);
   }
 
-  if (arg->type == DATA_UPLOAD_S3) {
-    if (code == 0 && (code = getCheckpointDataMeta(arg->taskId, path, toDelFiles)) != 0) {
-      stError("s-task:%s failed to get checkpointId:%" PRId64 " meta", taskStr, arg->chkpId);
+  if (pParam->type == DATA_UPLOAD_S3) {
+    if (code == 0 && (code = getCheckpointDataMeta(pParam->taskId, path, toDelFiles)) != 0) {
+      stError("s-task:%s failed to get checkpointData for checkpointId:%" PRId64 " meta", taskStr, pParam->chkpId);
     }
   }
 
-  if (code == 0 && (code = streamTaskBackupCheckpoint(arg->taskId, path)) != 0) {
-    stError("s-task:%s failed to upload checkpointId:%" PRId64, taskStr, arg->chkpId);
+  if (code == TSDB_CODE_SUCCESS) {
+    code = streamTaskBackupCheckpoint(pParam->taskId, path);
+    if (code != TSDB_CODE_SUCCESS) {
+      stError("s-task:%s failed to upload checkpoint data:%s, checkpointId:%" PRId64, taskStr, path, pParam->chkpId);
+    } else {
+      stDebug("s-task:%s backup checkpointId:%"PRId64" to remote succ", taskStr, pParam->chkpId);
+    }
   }
 
-  taskReleaseDb(arg->dbRefId);
+  taskReleaseDb(pParam->dbRefId);
 
   if (code == 0) {
-    for (int i = 0; i < taosArrayGetSize(toDelFiles); i++) {
-      char* p = taosArrayGetP(toDelFiles, i);
-      code = deleteCheckpointFile(arg->taskId, p);
-      stDebug("s-task:%s try to del file: %s", taskStr, p);
+    int32_t size = taosArrayGetSize(toDelFiles);
+    stDebug("s-task:%s remove redundant %d files", taskStr, size);
+
+    for (int i = 0; i < size; i++) {
+      char* pName = taosArrayGetP(toDelFiles, i);
+      code = deleteCheckpointFile(pParam->taskId, pName);
+      stDebug("s-task:%s try to del file: %s", taskStr, pName);
       if (code != 0) {
         break;
       }
@@ -396,15 +405,18 @@ int32_t uploadCheckpointData(void* param) {
   }
 
   taosArrayDestroyP(toDelFiles, taosMemoryFree);
+
+  stDebug("s-task:%s remove local checkpoint dir:%s", taskStr, path);
   taosRemoveDir(path);
   taosMemoryFree(path);
-  taosMemoryFree(arg->taskId);
-  taosMemoryFree(arg);
+
+  taosMemoryFree(pParam->taskId);
+  taosMemoryFree(pParam);
 
   return code;
 }
 
-int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t chkpId, char* taskId) {
+int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointId, char* taskId) {
   ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
   if (type == DATA_UPLOAD_DISABLE) {
     return 0;
@@ -417,7 +429,7 @@ int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t chkpId, cha
   SAsyncUploadArg* arg = taosMemoryCalloc(1, sizeof(SAsyncUploadArg));
   arg->type = type;
   arg->taskId = taosStrdup(taskId);
-  arg->chkpId = chkpId;
+  arg->chkpId = checkpointId;
   arg->pTask = pTask;
   arg->dbRefId = taskGetDBRef(pTask->pBackend);
   arg->pMeta = pTask->pMeta;
@@ -613,6 +625,7 @@ int32_t deleteCheckpoint(const char* id) {
 int32_t deleteCheckpointFile(const char* id, const char* name) {
   char object[128] = {0};
   snprintf(object, sizeof(object), "%s/%s", id, name);
+
   char* tmp = object;
   s3DeleteObjects((const char**)&tmp, 1);
   return 0;

From 370c25512822ad679cd9ce9bc91b1a4845d49e63 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 11 May 2024 10:27:55 +0800
Subject: [PATCH 003/132] refactor: do some internal refactor.

---
 source/common/src/rsync.c                 | 22 +++++++++++++---------
 source/libs/stream/src/streamCheckpoint.c |  3 +--
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index 867b407296..2324829dbb 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -153,16 +153,16 @@ void startRsync() {
 }
 
 int32_t uploadRsync(const char* id, const char* path) {
+  int64_t st = taosGetTimestampMs();
+  char    command[PATH_MAX] = {0};
+
 #ifdef WINDOWS
   char pathTransform[PATH_MAX] = {0};
   changeDirFromWindowsToLinux(path, pathTransform);
-#endif
 
-  char command[PATH_MAX] = {0};
-#ifdef WINDOWS
-  if(pathTransform[strlen(pathTransform) - 1] != '/'){
+  if(pathTransform[strlen(pathTransform) - 1] != '/') {
 #else
-  if(path[strlen(path) - 1] != '/'){
+  if (path[strlen(path) - 1] != '/') {
 #endif
     snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/",
 #ifdef WINDOWS
@@ -178,16 +178,20 @@ int32_t uploadRsync(const char* id, const char* path) {
 #else
              path
 #endif
-             , tsSnodeAddress, id);
+             ,
+             tsSnodeAddress, id);
   }
 
   int32_t code = execCommand(command);
-  if(code != 0){
-    uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+  if (code != 0) {
+    uError("[rsync] s-task:%s upload checkpoint data in:%s to %s failed, code:%d", id, path, tsSnodeAddress,
+           ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
     return -1;
   }
 
-  uDebug("[rsync] upload data:%s successful", id);
+  int64_t el = (taosGetTimestampMs() - st);
+  uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
+         tsSnodeAddress, el);
   return 0;
 }
 
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index a1e6838ac3..2f1771279f 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -368,8 +368,7 @@ int32_t uploadCheckpointData(void* param) {
   }
 
   if ((code = taskDbGenChkpUploadData(pParam->pTask->pBackend, ((SStreamMeta*)pParam->pMeta)->bkdChkptMgt,
-                                      pParam->chkpId,
-                                      (int8_t)(pParam->type), &path, toDelFiles)) != 0) {
+                                      pParam->chkpId, (int8_t)(pParam->type), &path, toDelFiles)) != 0) {
     stError("s-task:%s failed to gen upload checkpoint:%" PRId64, taskStr, pParam->chkpId);
   }
 

From f8d80612606b1db876c090dfe06b2782afe0c3eb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 11 May 2024 11:08:51 +0800
Subject: [PATCH 004/132] refactor: do some internal refactor.

---
 source/common/src/rsync.c                     | 4 ++--
 source/common/src/tdataformat.c               | 1 +
 source/libs/executor/src/timesliceoperator.c  | 1 +
 source/libs/stream/src/streamBackendRocksdb.c | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index 2324829dbb..35ae9450f1 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -184,8 +184,8 @@ int32_t uploadRsync(const char* id, const char* path) {
 
   int32_t code = execCommand(command);
   if (code != 0) {
-    uError("[rsync] s-task:%s upload checkpoint data in:%s to %s failed, code:%d", id, path, tsSnodeAddress,
-           ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
+           tsSnodeAddress, code, ERRNO_ERR_DATA);
     return -1;
   }
 
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 04ad00e1dc..f2d37bede6 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1320,6 +1320,7 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
         pVal->val = pSrc->pks[i].val;
       } else {
         pVal->nData = pSrc->pks[i].nData;
+        ASSERT(pSrc->pks[i].pData != NULL);
         memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
       }
     }
diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c
index 080fd6b914..ec133a1022 100644
--- a/source/libs/executor/src/timesliceoperator.c
+++ b/source/libs/executor/src/timesliceoperator.c
@@ -194,6 +194,7 @@ static void tRowGetKeyFromColData(int64_t ts, SColumnInfoData* pPkCol, int32_t r
   }
 }
 
+// only the timestamp is needed to complete the duplicated timestamp check.
 static bool checkDuplicateTimestamps(STimeSliceOperatorInfo* pSliceInfo, SColumnInfoData* pTsCol,
                                      SColumnInfoData* pPkCol, int32_t curIndex, int32_t rows) {
   int64_t currentTs = *(int64_t*)colDataGetData(pTsCol, curIndex);
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 60384c9ca1..455466f64c 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -539,7 +539,7 @@ int32_t rebuildDirFormCheckpoint(const char* path, char* key, int64_t chkptId, c
     taosMulMkDir(defaultPath);
   }
 
-  stDebug("prepare local dir:%s, checkpointId:%d, key:%s succ", defaultPath, chkptId, key);
+  stDebug("prepare local dir:%s, checkpointId:%" PRId64 ", key:%s succ", defaultPath, chkptId, key);
 
   char* chkptPath = taosMemoryCalloc(1, strlen(path) + 256);
   if (chkptId != 0) {

From d214dd5cdd7103a3a31a8f4a09c9998a556e43b4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 11 May 2024 16:19:11 +0800
Subject: [PATCH 005/132] fix(stream): fix error in downloading remote backup
 checkpoint data.

---
 source/common/src/rsync.c                     | 14 +++++---
 source/libs/stream/src/streamBackendRocksdb.c | 35 +++++++++----------
 2 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index 35ae9450f1..302f17942f 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -196,10 +196,14 @@ int32_t uploadRsync(const char* id, const char* path) {
 }
 
 int32_t downloadRsync(const char* id, const char* path) {
+  int64_t st = taosGetTimestampMs();
+  uDebug("[rsync] %s start to sync data from remote to local:%s", id, path);
+
 #ifdef WINDOWS
   char pathTransform[PATH_MAX] = {0};
   changeDirFromWindowsToLinux(path, pathTransform);
 #endif
+
   char command[PATH_MAX] = {0};
   snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
            tsSnodeAddress, id,
@@ -211,13 +215,15 @@ int32_t downloadRsync(const char* id, const char* path) {
            );
 
   int32_t code = execCommand(command);
+
+  int32_t el = taosGetTimestampMs() - st;
   if (code != 0) {
-    uError("[rsync] download checkpoint data failed, code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
-    return -1;
+    uError("[rsync] %s download checkpoint data:%s failed, code:%d," ERRNO_ERR_FORMAT, id, path, code, ERRNO_ERR_DATA);
+  } else {
+    uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el);
   }
 
-  uDebug("[rsync] download checkpoint data:%s successfully", id);
-  return 0;
+  return code;
 }
 
 int32_t deleteRsync(const char* id) {
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 455466f64c..1a05acf3eb 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -325,28 +325,27 @@ int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) {
   return complete == 1 ? 0 : -1;
 }
 
-int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkpPath, int64_t checkpointId, char* defaultPath) {
-  // impl later
+int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) {
   int32_t code = 0;
-  if (taosIsDir(chkpPath)) {
-    taosRemoveDir(chkpPath);
+  if (taosIsDir(chkptPath)) {
+    taosRemoveDir(chkptPath);
+    stDebug("remove local checkpoint data dir:%s succ", chkptPath);
   }
 
   if (taosIsDir(defaultPath)) {
     taosRemoveDir(defaultPath);
+    taosMulMkDir(defaultPath);
+    stDebug("clear local backend dir:%s succ", defaultPath);
   }
 
-  code = streamTaskDownloadCheckpointData(key, chkpPath);
+  code = streamTaskDownloadCheckpointData(key, chkptPath);
   if (code != 0) {
     stError("failed to download checkpoint data:%s", key);
     return code;
   }
 
-  stDebug("download backup checkpoint data into:%s, checkpointId:%" PRId64 ", %s", chkpPath, checkpointId, key);
-
-  code = backendCopyFiles(chkpPath, defaultPath);
-
-  return code;
+  stDebug("download remote checkpoint data for checkpointId:%" PRId64 ", %s", checkpointId, key);
+  return backendCopyFiles(chkptPath, defaultPath);
 }
 
 int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
@@ -456,7 +455,7 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) {
       code = copyFiles_hardlink(srcName, dstName, 0);
       if (code != 0) {
         code = TAOS_SYSTEM_ERROR(code);
-        stError("failed to hardlink file, detail:%s to %s, reason:%s", srcName, dstName, tstrerror(code));
+        stError("failed to hard link file, detail:%s to %s, reason:%s", srcName, dstName, tstrerror(code));
         goto _ERROR;
       } else {
         stDebug("succ hard link file:%s to %s", srcName, dstName);
@@ -485,14 +484,13 @@ int32_t backendCopyFiles(const char* src, const char* dst) {
   return backendFileCopyFilesImpl(src, dst);
 }
 
-static int32_t rebuildFromLocalCheckpoint(char* pTaskIdStr, const char* checkpointPath, int64_t chkpId, const char* defaultPath) {
+static int32_t rebuildFromLocalCheckpoint(char* pTaskIdStr, const char* checkpointPath, int64_t chkptId, const char* defaultPath) {
   int32_t code = 0;
 
   if (taosIsDir(defaultPath)) {
     taosRemoveDir(defaultPath);
     taosMkDir(defaultPath);
-
-    stInfo("clear task backend path:%s, done", defaultPath);
+    stInfo("clear task backend dir:%s, done", defaultPath);
   }
 
   if (taosIsDir(checkpointPath) && isValidCheckpoint(checkpointPath)) {
@@ -506,11 +504,12 @@ static int32_t rebuildFromLocalCheckpoint(char* pTaskIdStr, const char* checkpoi
               pTaskIdStr, checkpointPath, tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath);
       code = TSDB_CODE_SUCCESS;
     } else {
-      stInfo("%s start to restart stream backend at checkpoint path: %s", pTaskIdStr, checkpointPath);
+      stInfo("%s copy checkpoint data from:%s to:%s succ, try to start stream backend", pTaskIdStr, checkpointPath,
+             defaultPath);
     }
   } else {
     code = TSDB_CODE_FAILED;
-    stError("%s not valid checkpoint path/data in:%s", pTaskIdStr, checkpointPath);
+    stError("%s no valid checkpoint data for checkpointId:%" PRId64 " in %s", pTaskIdStr, chkptId, checkpointPath);
   }
 
   return code;
@@ -522,7 +521,6 @@ int32_t rebuildFromlocalDefault(char* key, char* chkpPath, int64_t chkpId, char*
 }
 
 int32_t rebuildDirFormCheckpoint(const char* path, char* key, int64_t chkptId, char** dbPrefixPath, char** dbPath) {
-  // impl later
   int32_t code = 0;
 
   char* prefixPath = taosMemoryCalloc(1, strlen(path) + 128);
@@ -538,8 +536,7 @@ int32_t rebuildDirFormCheckpoint(const char* path, char* key, int64_t chkptId, c
   if (!taosIsDir(defaultPath)) {
     taosMulMkDir(defaultPath);
   }
-
-  stDebug("prepare local dir:%s, checkpointId:%" PRId64 ", key:%s succ", defaultPath, chkptId, key);
+  stDebug("local default dir:%s, checkpointId:%" PRId64 ", key:%s succ", defaultPath, chkptId, key);
 
   char* chkptPath = taosMemoryCalloc(1, strlen(path) + 256);
   if (chkptId != 0) {

From 3dfffe91130dbc57893bb2fd0f472817c3621a18 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 13 May 2024 15:31:26 +0800
Subject: [PATCH 006/132] fix(stream): async load and start stream tasks.

---
 include/libs/stream/tstream.h                 |  4 +++
 source/common/src/rsync.c                     | 36 ++++++++++++-------
 source/dnode/snode/src/snode.c                | 11 +++---
 source/dnode/vnode/src/tq/tq.c                |  7 ++--
 source/dnode/vnode/src/tq/tqMeta.c            | 30 +---------------
 source/dnode/vnode/src/tqCommon/tqCommon.c    | 10 ++++--
 source/dnode/vnode/src/vnd/vnodeSync.c        | 26 +++++++-------
 source/libs/stream/src/streamBackendRocksdb.c |  2 +-
 source/libs/stream/src/streamCheckpoint.c     | 12 +++----
 source/libs/stream/src/streamSched.c          |  6 +++-
 10 files changed, 69 insertions(+), 75 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 9d5b7bc6f1..bddc76cfb5 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -58,6 +58,10 @@ extern "C" {
 #define STREAM_EXEC_T_STOP_ALL_TASKS    (-5)
 #define STREAM_EXEC_T_RESUME_TASK       (-6)
 #define STREAM_EXEC_T_ADD_FAILED_TASK   (-7)
+// the load and start stream task should be executed after snode has started successfully, since the load of stream
+// tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources.
+#define STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS (-8)
+#define STREAM_EXEC_T_LOAD_ALL_TASKS           (-9)
 
 typedef struct SStreamTask      SStreamTask;
 typedef struct SStreamQueue     SStreamQueue;
diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index 302f17942f..149c36cec7 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -130,6 +130,7 @@ void startRsync() {
     uError("[rsync] build checkpoint backup dir failed, dir:%s,"ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA);
     return;
   }
+
   removeEmptyDir();
 
   char confDir[PATH_MAX] = {0};
@@ -186,18 +187,20 @@ int32_t uploadRsync(const char* id, const char* path) {
   if (code != 0) {
     uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
            tsSnodeAddress, code, ERRNO_ERR_DATA);
-    return -1;
+  } else {
+    int64_t el = (taosGetTimestampMs() - st);
+    uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
+           tsSnodeAddress, el);
   }
 
-  int64_t el = (taosGetTimestampMs() - st);
-  uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
-         tsSnodeAddress, el);
-  return 0;
+  return code;
 }
 
 int32_t downloadRsync(const char* id, const char* path) {
   int64_t st = taosGetTimestampMs();
-  uDebug("[rsync] %s start to sync data from remote to local:%s", id, path);
+  int32_t MAX_RETRY = 60;
+  int32_t times = 0;
+  int32_t code = 0;
 
 #ifdef WINDOWS
   char pathTransform[PATH_MAX] = {0};
@@ -205,7 +208,7 @@ int32_t downloadRsync(const char* id, const char* path) {
 #endif
 
   char command[PATH_MAX] = {0};
-  snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
+  snprintf(command, PATH_MAX, "rsync -av --debug=all --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
            tsSnodeAddress, id,
 #ifdef WINDOWS
            pathTransform
@@ -214,13 +217,20 @@ int32_t downloadRsync(const char* id, const char* path) {
 #endif
            );
 
-  int32_t code = execCommand(command);
+  uDebug("[rsync] %s start to sync data from remote to local:%s, %s", id, path, command);
 
-  int32_t el = taosGetTimestampMs() - st;
-  if (code != 0) {
-    uError("[rsync] %s download checkpoint data:%s failed, code:%d," ERRNO_ERR_FORMAT, id, path, code, ERRNO_ERR_DATA);
-  } else {
-    uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el);
+  while(times++ < MAX_RETRY) {
+
+    code = execCommand(command);
+    if (code != TSDB_CODE_SUCCESS) {
+      uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, code:%d," ERRNO_ERR_FORMAT, id, path, code,
+             ERRNO_ERR_DATA);
+      taosSsleep(1);
+    } else {
+      int32_t el = taosGetTimestampMs() - st;
+      uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el);
+      break;
+    }
   }
 
   return code;
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 602264be73..e1b51e3c1a 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -86,6 +86,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
     return NULL;
   }
 
+  stopRsync();
+  startRsync();
+
   pSnode->msgCb = pOption->msgCb;
   pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback);
   if (pSnode->pMeta == NULL) {
@@ -93,11 +96,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
     goto FAIL;
   }
 
-  streamMetaLoadAllTasks(pSnode->pMeta);
-
-  stopRsync();
-  startRsync();
-
   return pSnode;
 
 FAIL:
@@ -106,8 +104,7 @@ FAIL:
 }
 
 int32_t sndInit(SSnode *pSnode) {
-  streamMetaResetTaskStatus(pSnode->pMeta);
-  streamMetaStartAllTasks(pSnode->pMeta);
+  streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS);
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index b75d517997..19c0f60063 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -92,8 +92,6 @@ int32_t tqInitialize(STQ* pTq) {
     return -1;
   }
 
-  streamMetaLoadAllTasks(pTq->pStreamMeta);
-
   if (tqMetaTransform(pTq) < 0) {
     return -1;
   }
@@ -800,6 +798,11 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) {
 }
 
 int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
+  if (!pTq->pVnode->restored) {
+    tqDebug("vgId:%d not restored, ignore the stream task deploy msg", TD_VID(pTq->pVnode));
+    return TSDB_CODE_SUCCESS;
+  }
+
   return tqStreamTaskProcessDeployReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, sversion, msg, msgLen,
                                       vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored);
 }
diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c
index 76322c527f..404cbf26dd 100644
--- a/source/dnode/vnode/src/tq/tqMeta.c
+++ b/source/dnode/vnode/src/tq/tqMeta.c
@@ -178,6 +178,7 @@ int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
       goto END;
     }
   }
+
 END:
   tdbFree(pKey);
   tdbFree(pVal);
@@ -514,35 +515,6 @@ int32_t tqMetaTransform(STQ* pTq) {
   return code;
 }
 
-//int32_t tqMetaRestoreHandle(STQ* pTq) {
-//  int  code = 0;
-//  TBC* pCur = NULL;
-//  if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) {
-//    return -1;
-//  }
-//
-//  void*    pKey = NULL;
-//  int      kLen = 0;
-//  void*    pVal = NULL;
-//  int      vLen = 0;
-//
-//  tdbTbcMoveToFirst(pCur);
-//
-//  while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
-//    STqHandle handle = {0};
-//    code = restoreHandle(pTq, pVal, vLen, &handle);
-//    if (code < 0) {
-//      tqDestroyTqHandle(&handle);
-//      break;
-//    }
-//  }
-//
-//  tdbFree(pKey);
-//  tdbFree(pVal);
-//  tdbTbcClose(pCur);
-//  return code;
-//}
-
 int32_t tqMetaGetHandle(STQ* pTq, const char* key) {
   void*    pVal = NULL;
   int      vLen = 0;
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 91de290e6a..cbd047cf88 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -158,10 +158,9 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
   STaskId       id = {.streamId = req.streamId, .taskId = req.taskId};
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
   if (ppTask == NULL || *ppTask == NULL) {
-    tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped", vgId, req.taskId);
+    tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId, req.taskId);
     rsp.code = TSDB_CODE_SUCCESS;
     streamMetaWUnLock(pMeta);
-
     taosArrayDestroy(req.pNodeList);
     return rsp.code;
   }
@@ -739,6 +738,13 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
   } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
     int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
     return code;
+  } else if (type == STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS) {
+    streamMetaLoadAllTasks(pMeta);
+    int32_t code = streamMetaStartAllTasks(pMeta);
+    return code;
+  } else if (type == STREAM_EXEC_T_LOAD_ALL_TASKS) {
+    streamMetaLoadAllTasks(pMeta);
+    return 0;
   } else if (type == STREAM_EXEC_T_RESUME_TASK) {  // task resume to run after idle for a while
     SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 166a230c76..d2c20500be 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -576,24 +576,22 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
     if (tsDisableStream) {
       vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId);
     } else {
-      vInfo("vgId:%d sync restore finished, start to launch stream task(s)", pVnode->config.vgId);
-      int32_t numOfTasks = tqStreamTasksGetTotalNum(pMeta);
-      if (numOfTasks > 0) {
-        if (pMeta->startInfo.startAllTasks == 1) {
-          pMeta->startInfo.restartCount += 1;
-          tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
-                  pMeta->startInfo.restartCount);
-        } else {
-          pMeta->startInfo.startAllTasks = 1;
+      vInfo("vgId:%d sync restore finished, start to load and launch stream task(s)", pVnode->config.vgId);
+      if (pMeta->startInfo.startAllTasks == 1) {
+        pMeta->startInfo.restartCount += 1;
+        tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
+                pMeta->startInfo.restartCount);
+      } else {
+        pMeta->startInfo.startAllTasks = 1;
+        streamMetaWUnLock(pMeta);
 
-          streamMetaWUnLock(pMeta);
-          tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false);
-          return;
-        }
+        streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS);
+        return;
       }
     }
   } else {
-    vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId);
+    vInfo("vgId:%d, sync restore finished, load stream tasks, not start tasks since not leader", vgId);
+    streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_ALL_TASKS);
   }
 
   streamMetaWUnLock(pMeta);
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 1a05acf3eb..dd897dc431 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -490,7 +490,7 @@ static int32_t rebuildFromLocalCheckpoint(char* pTaskIdStr, const char* checkpoi
   if (taosIsDir(defaultPath)) {
     taosRemoveDir(defaultPath);
     taosMkDir(defaultPath);
-    stInfo("clear task backend dir:%s, done", defaultPath);
+    stInfo("clear local backend dir:%s, done", defaultPath);
   }
 
   if (taosIsDir(checkpointPath) && isValidCheckpoint(checkpointPath)) {
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 2f1771279f..1b6c9cc791 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -30,7 +30,7 @@ typedef struct {
 
 static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName);
 static int32_t deleteCheckpointFile(const char* id, const char* name);
-static int32_t streamTaskBackupCheckpoint(const char* id, const char* path);
+static int32_t streamTaskUploadCheckpoint(const char* id, const char* path);
 static int32_t deleteCheckpoint(const char* id);
 static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName);
 static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask);
@@ -321,7 +321,7 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l
 
   int32_t code = downloadCheckpointDataByName(id, "META", file);
   if (code != 0) {
-    stDebug("chkp failed to download meta file:%s", file);
+    stDebug("%s chkp failed to download meta file:%s", id, file);
     taosMemoryFree(file);
     return code;
   }
@@ -379,7 +379,7 @@ int32_t uploadCheckpointData(void* param) {
   }
 
   if (code == TSDB_CODE_SUCCESS) {
-    code = streamTaskBackupCheckpoint(pParam->taskId, path);
+    code = streamTaskUploadCheckpoint(pParam->taskId, path);
     if (code != TSDB_CODE_SUCCESS) {
       stError("s-task:%s failed to upload checkpoint data:%s, checkpointId:%" PRId64, taskStr, path, pParam->chkpId);
     } else {
@@ -562,9 +562,9 @@ ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType() {
   }
 }
 
-int32_t streamTaskBackupCheckpoint(const char* id, const char* path) {
+int32_t streamTaskUploadCheckpoint(const char* id, const char* path) {
   if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) {
-    stError("streamTaskBackupCheckpoint parameters invalid");
+    stError("invalid parameters in upload checkpoint, %s", id);
     return -1;
   }
 
@@ -580,7 +580,7 @@ int32_t streamTaskBackupCheckpoint(const char* id, const char* path) {
 // fileName:  CURRENT
 int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName) {
   if (id == NULL || fname == NULL || strlen(id) == 0 || strlen(fname) == 0 || strlen(fname) >= PATH_MAX) {
-    stError("uploadCheckpointByName parameters invalid");
+    stError("down load checkpoint data parameters invalid");
     return -1;
   }
 
diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c
index 52e7431e70..9bd12a4fd8 100644
--- a/source/libs/stream/src/streamSched.c
+++ b/source/libs/stream/src/streamSched.c
@@ -52,7 +52,11 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3
     return -1;
   }
 
-  stDebug("vgId:%d create msg to start stream task:0x%x, exec type:%d", vgId, taskId, execType);
+  if (streamId != 0) {
+    stDebug("vgId:%d create msg to start stream task:0x%x, exec type:%d", vgId, taskId, execType);
+  } else {
+    stDebug("vgId:%d create msg to exec, type:%d", vgId, execType);
+  }
 
   pRunReq->head.vgId = vgId;
   pRunReq->streamId = streamId;

From b7e7de0354709de9f422d071464085e935b3227a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 13 May 2024 16:45:38 +0800
Subject: [PATCH 007/132] fix(stream): set the stream task load flag.

---
 include/libs/stream/tstream.h          |  1 +
 source/dnode/vnode/src/vnd/vnodeSync.c | 25 ++++++++++++++++++++-----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index bddc76cfb5..9e376b9792 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -491,6 +491,7 @@ typedef struct SStreamMeta {
   int32_t         vgId;
   int64_t         stage;
   int32_t         role;
+  bool            taskLoadFlag;
   bool            closeFlag;
   bool            sendMsgBeforeClosing;  // send hb to mnode before close all tasks when switch to follower.
   STaskStartInfo  startInfo;
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index d2c20500be..00203c7bb1 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -576,22 +576,37 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
     if (tsDisableStream) {
       vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId);
     } else {
-      vInfo("vgId:%d sync restore finished, start to load and launch stream task(s)", pVnode->config.vgId);
+      vInfo("vgId:%d sync restore finished, start to load and launch stream task(s)", vgId);
       if (pMeta->startInfo.startAllTasks == 1) {
         pMeta->startInfo.restartCount += 1;
-        tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
+        vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
                 pMeta->startInfo.restartCount);
       } else {
         pMeta->startInfo.startAllTasks = 1;
+
+        bool loadTaskInfo = pMeta->taskLoadFlag;
+        pMeta->taskLoadFlag = true;
         streamMetaWUnLock(pMeta);
 
-        streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS);
+        if (loadTaskInfo) {
+          tqInfo("vgId:%d stream task already loaded, start them", vgId);
+          streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_START_ALL_TASKS);
+        } else {
+          tqInfo("vgId:%d start load and launch stream task(s)", vgId);
+          streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS);
+        }
+
         return;
       }
     }
   } else {
-    vInfo("vgId:%d, sync restore finished, load stream tasks, not start tasks since not leader", vgId);
-    streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_ALL_TASKS);
+    if (!pMeta->taskLoadFlag) {
+      pMeta->taskLoadFlag = true;
+      vInfo("vgId:%d, sync restore finished, load stream tasks, not start tasks since not leader", vgId);
+      streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_ALL_TASKS);
+    } else {
+      vInfo("vgId:%d, sync restore finished, not load stream tasks since already loaded for follower");
+    }
   }
 
   streamMetaWUnLock(pMeta);

From fb248b2682257911ea6def4d942dcdb1da4eecb7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 13 May 2024 17:26:39 +0800
Subject: [PATCH 008/132] fix(stream): not save the stream task for rsma.

---
 source/libs/stream/src/streamCheckpoint.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 1b6c9cc791..389eccd66d 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -469,8 +469,8 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
     }
   }
 
-  // clear the checkpoint info, and commit the newest checkpoint info if all works are done successfully
-  if (code == TSDB_CODE_SUCCESS) {
+  // update the latest checkpoint info if all works are done successfully, for rsma, the pMsgCb is null.
+  if (code == TSDB_CODE_SUCCESS && (pTask->pMsgCb != NULL)) {
     STaskId* pHTaskId = &pTask->hTaskInfo.id;
     code = streamBuildAndSendCheckpointUpdateMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, pHTaskId, &pTask->chkInfo,
                                                  dropRelHTask);

From 6898eba6edf9cb493413e1542c06239739ca96f8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 14 May 2024 14:34:31 +0800
Subject: [PATCH 009/132] fix(stream): expand stream tasks are divided into two
 phase, the first is to build stream task and then expand stream task before
 start stream tasks.

---
 include/dnode/vnode/tqCommon.h                |  2 +-
 include/libs/executor/storageapi.h            |  2 +-
 include/libs/function/function.h              |  1 +
 include/libs/stream/streamState.h             |  2 +-
 include/libs/stream/tstream.h                 | 13 ++--
 source/common/src/rsync.c                     |  2 +-
 source/dnode/snode/src/snode.c                | 10 +--
 source/dnode/vnode/src/tq/tq.c                | 37 +++++-----
 source/dnode/vnode/src/tq/tqRead.c            |  2 +-
 source/dnode/vnode/src/tqCommon/tqCommon.c    | 26 ++++---
 source/dnode/vnode/src/vnd/vnodeSync.c        | 23 +-----
 source/libs/stream/inc/streamBackendRocksdb.h |  2 +-
 source/libs/stream/inc/streamInt.h            |  2 +-
 source/libs/stream/src/streamBackendRocksdb.c | 73 ++++++++++---------
 source/libs/stream/src/streamCheckpoint.c     | 11 ++-
 source/libs/stream/src/streamMeta.c           | 25 +++++--
 source/libs/stream/src/streamState.c          |  4 +-
 17 files changed, 122 insertions(+), 115 deletions(-)

diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h
index 451f9a00eb..0cde499a6b 100644
--- a/include/dnode/vnode/tqCommon.h
+++ b/include/dnode/vnode/tqCommon.h
@@ -40,7 +40,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
 int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
 int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
 
-int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode);
+int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta);
 void    tqSetRestoreVersionInfo(SStreamTask* pTask);
 
 #endif  // TDENGINE_TQ_COMMON_H
diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h
index ec92bd56dd..7042ec2d15 100644
--- a/include/libs/executor/storageapi.h
+++ b/include/libs/executor/storageapi.h
@@ -410,7 +410,7 @@ typedef struct SStateStore {
   void (*streamFileStateClear)(struct SStreamFileState* pFileState);
   bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
 
-  SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
+  SStreamState* (*streamStateOpen)(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
   void (*streamStateClose)(SStreamState* pState, bool remove);
   int32_t (*streamStateBegin)(SStreamState* pState);
   int32_t (*streamStateCommit)(SStreamState* pState);
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 0afda2e160..87bbe21133 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -150,6 +150,7 @@ typedef struct SBackendCfWrapper {
   int64_t        backendId;
   char           idstr[64];
 } SBackendCfWrapper;
+
 typedef struct STdbState {
   SBackendCfWrapper *pBackendCfWrapper;
   int64_t            backendCfWrapperId;
diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h
index 7813b2cc9a..ae5a733ae9 100644
--- a/include/libs/stream/streamState.h
+++ b/include/libs/stream/streamState.h
@@ -29,7 +29,7 @@ extern "C" {
 
 #include "storageapi.h"
 
-SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
+SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
 void          streamStateClose(SStreamState* pState, bool remove);
 int32_t       streamStateBegin(SStreamState* pState);
 int32_t       streamStateCommit(SStreamState* pState);
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 9e376b9792..1f5aa46f49 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -61,7 +61,6 @@ extern "C" {
 // the load and start stream task should be executed after snode has started successfully, since the load of stream
 // tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources.
 #define STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS (-8)
-#define STREAM_EXEC_T_LOAD_ALL_TASKS           (-9)
 
 typedef struct SStreamTask      SStreamTask;
 typedef struct SStreamQueue     SStreamQueue;
@@ -156,8 +155,6 @@ typedef enum EStreamTaskEvent {
   TASK_EVENT_DROPPING = 0xA,
 } EStreamTaskEvent;
 
-typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
-
 typedef void    FTbSink(SStreamTask* pTask, void* vnode, void* data);
 typedef void    FSmaSink(void* vnode, int64_t smaId, const SArray* data);
 typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver);
@@ -491,7 +488,6 @@ typedef struct SStreamMeta {
   int32_t         vgId;
   int64_t         stage;
   int32_t         role;
-  bool            taskLoadFlag;
   bool            closeFlag;
   bool            sendMsgBeforeClosing;  // send hb to mnode before close all tasks when switch to follower.
   STaskStartInfo  startInfo;
@@ -522,6 +518,9 @@ typedef struct STaskUpdateEntry {
   int32_t transId;
 } STaskUpdateEntry;
 
+typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
+typedef int32_t (*__stream_task_expand_fn)(struct SStreamTask* pTask);
+
 SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
                             SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5);
 int32_t      tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
@@ -675,7 +674,7 @@ int32_t streamTaskReleaseState(SStreamTask* pTask);
 int32_t streamTaskReloadState(SStreamTask* pTask);
 void    streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
 void    streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
-int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key);
+int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key);
 bool    streamTaskIsSinkTask(const SStreamTask* pTask);
 
 void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
@@ -723,9 +722,9 @@ void    streamMetaResetStartInfo(STaskStartInfo* pMeta);
 SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta);
 void    streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader);
 void    streamMetaLoadAllTasks(SStreamMeta* pMeta);
-int32_t streamMetaStartAllTasks(SStreamMeta* pMeta);
+int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn fn);
 int32_t streamMetaStopAllTasks(SStreamMeta* pMeta);
-int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
+int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, __stream_task_expand_fn fn);
 bool    streamMetaAllTasksReady(const SStreamMeta* pMeta);
 
 // timer
diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index 149c36cec7..c4d14a6c2c 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -217,7 +217,7 @@ int32_t downloadRsync(const char* id, const char* path) {
 #endif
            );
 
-  uDebug("[rsync] %s start to sync data from remote to local:%s, %s", id, path, command);
+  uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command);
 
   while(times++ < MAX_RETRY) {
 
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index e1b51e3c1a..ac10aa83a4 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -45,15 +45,10 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
+
   pTask->pBackend = NULL;
-
   streamTaskOpenAllUpstreamInput(pTask);
 
-  code = tqExpandStreamTask(pTask, pSnode->pMeta, NULL);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
-  }
-
   streamTaskResetUpstreamStageInfo(pTask);
   streamSetupScheduleTrigger(pTask);
 
@@ -96,6 +91,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
     goto FAIL;
   }
 
+  streamMetaLoadAllTasks(pSnode->pMeta);
   return pSnode;
 
 FAIL:
@@ -104,7 +100,7 @@ FAIL:
 }
 
 int32_t sndInit(SSnode *pSnode) {
-  streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS);
+  streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS);
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 19c0f60063..a59a235c50 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -92,6 +92,8 @@ int32_t tqInitialize(STQ* pTq) {
     return -1;
   }
 
+  streamMetaLoadAllTasks(pTq->pStreamMeta);
+
   if (tqMetaTransform(pTq) < 0) {
     return -1;
   }
@@ -715,17 +717,18 @@ static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); }
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   int32_t vgId = TD_VID(pTq->pVnode);
-  tqDebug("s-task:0x%x start to expand task", pTask->id.taskId);
+  tqDebug("s-task:0x%x start to build task", pTask->id.taskId);
 
   int32_t code = streamTaskInit(pTask, pTq->pStreamMeta, &pTq->pVnode->msgCb, nextProcessVer);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
 
-  code = tqExpandStreamTask(pTask, pTq->pStreamMeta, pTq->pVnode);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
-  }
+  pTask->pBackend = NULL;
+//  code = tqExpandStreamTask(pTask, pTq->pStreamMeta);
+//  if (code != TSDB_CODE_SUCCESS) {
+//    return code;
+//  }
 
   // sink
   STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
@@ -768,7 +771,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus);
 
   if (pTask->info.fillHistory) {
-    tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
+    tqInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
            " nextProcessVer:%" PRId64
            " child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x "
            "trigger:%" PRId64 " ms, inputVer:%" PRId64,
@@ -777,7 +780,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
            (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam, nextProcessVer);
   } else {
     tqInfo(
-        "vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
+        "vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
         " nextProcessVer:%" PRId64
         " child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64
         " ms, inputVer:%" PRId64,
@@ -798,10 +801,10 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) {
 }
 
 int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
-  if (!pTq->pVnode->restored) {
-    tqDebug("vgId:%d not restored, ignore the stream task deploy msg", TD_VID(pTq->pVnode));
-    return TSDB_CODE_SUCCESS;
-  }
+//  if (!pTq->pVnode->restored) {
+//    tqDebug("vgId:%d not restored, ignore the stream task deploy msg", TD_VID(pTq->pVnode));
+//    return TSDB_CODE_SUCCESS;
+//  }
 
   return tqStreamTaskProcessDeployReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, sversion, msg, msgLen,
                                       vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored);
@@ -1016,12 +1019,12 @@ int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) {
   int32_t vgId = TD_VID(pTq->pVnode);
   SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
 
-  if (!pTq->pVnode->restored) {
-    tqDebug("vgId:%d update-checkpoint-info msg received during restoring, checkpointId:%" PRId64
-            ", transId:%d s-task:0x%x ignore it",
-            vgId, pReq->checkpointId, pReq->transId, pReq->taskId);
-    return TSDB_CODE_SUCCESS;
-  }
+//  if (!pTq->pVnode->restored) {
+//    tqDebug("vgId:%d update-checkpoint-info msg received during restoring, checkpointId:%" PRId64
+//            ", transId:%d s-task:0x%x ignore it",
+//            vgId, pReq->checkpointId, pReq->transId, pReq->taskId);
+//    return TSDB_CODE_SUCCESS;
+//  }
 
   return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, msg, msgLen);
 }
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 516a47606b..7224657b73 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -1089,7 +1089,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
     }
 
     SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && (pTask->exec.pExecutor != NULL)) {
       int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd);
       if (code != 0) {
         tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr);
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index cbd047cf88..dbda3a4541 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -39,9 +39,13 @@ static void restoreStreamTaskId(SStreamTask* pTask, STaskId* pId) {
   pTask->id.streamId = pId->streamId;
 }
 
-int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode) {
-  int32_t vgId = pMeta->vgId;
-  STaskId taskId = {0};
+int32_t tqExpandStreamTask(SStreamTask* pTask) {
+  SStreamMeta* pMeta = pTask->pMeta;
+  int32_t      vgId = pMeta->vgId;
+  STaskId      taskId = {0};
+  int64_t      st = taosGetTimestampMs();
+
+  tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId);
 
   if (pTask->info.fillHistory) {
     taskId = replaceStreamTaskId(pTask);
@@ -67,7 +71,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode)
   };
 
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    handle.vnode = pVnode;
+    handle.vnode = ((STQ*)pMeta->ahandle)->pVnode;
     handle.initTqReader = 1;
   } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
     handle.numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList);
@@ -84,6 +88,9 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode)
     qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId);
   }
 
+  double el = (taosGetTimestampMs() - st) / 1000.0;
+  tqDebug("s-task:%s vgId:%d expand stream task completed, elapsed time:%.2fsec", pTask->id.idStr, vgId, el);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -706,7 +713,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
     streamMetaResetTaskStatus(pMeta);
     streamMetaWUnLock(pMeta);
 
-    streamMetaStartAllTasks(pMeta);
+    streamMetaStartAllTasks(pMeta, tqExpandStreamTask);
   } else {
     streamMetaResetStartInfo(&pMeta->startInfo);
     streamMetaWUnLock(pMeta);
@@ -724,10 +731,10 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
   int32_t vgId = pMeta->vgId;
 
   if (type == STREAM_EXEC_T_START_ONE_TASK) {
-    streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
+    streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId, tqExpandStreamTask);
     return 0;
   } else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
-    streamMetaStartAllTasks(pMeta);
+    streamMetaStartAllTasks(pMeta, tqExpandStreamTask);
     return 0;
   } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
     restartStreamTasks(pMeta, isLeader);
@@ -740,11 +747,8 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
     return code;
   } else if (type == STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS) {
     streamMetaLoadAllTasks(pMeta);
-    int32_t code = streamMetaStartAllTasks(pMeta);
+    int32_t code = streamMetaStartAllTasks(pMeta, tqExpandStreamTask);
     return code;
-  } else if (type == STREAM_EXEC_T_LOAD_ALL_TASKS) {
-    streamMetaLoadAllTasks(pMeta);
-    return 0;
   } else if (type == STREAM_EXEC_T_RESUME_TASK) {  // task resume to run after idle for a while
     SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 00203c7bb1..8f28871e3b 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -576,37 +576,22 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
     if (tsDisableStream) {
       vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId);
     } else {
-      vInfo("vgId:%d sync restore finished, start to load and launch stream task(s)", vgId);
+      vInfo("vgId:%d sync restore finished, start to launch stream task(s)", vgId);
       if (pMeta->startInfo.startAllTasks == 1) {
         pMeta->startInfo.restartCount += 1;
         vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
                 pMeta->startInfo.restartCount);
       } else {
         pMeta->startInfo.startAllTasks = 1;
-
-        bool loadTaskInfo = pMeta->taskLoadFlag;
-        pMeta->taskLoadFlag = true;
         streamMetaWUnLock(pMeta);
 
-        if (loadTaskInfo) {
-          tqInfo("vgId:%d stream task already loaded, start them", vgId);
-          streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_START_ALL_TASKS);
-        } else {
-          tqInfo("vgId:%d start load and launch stream task(s)", vgId);
-          streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS);
-        }
-
+        tqInfo("vgId:%d stream task already loaded, start them", vgId);
+        streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_START_ALL_TASKS);
         return;
       }
     }
   } else {
-    if (!pMeta->taskLoadFlag) {
-      pMeta->taskLoadFlag = true;
-      vInfo("vgId:%d, sync restore finished, load stream tasks, not start tasks since not leader", vgId);
-      streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_LOAD_ALL_TASKS);
-    } else {
-      vInfo("vgId:%d, sync restore finished, not load stream tasks since already loaded for follower");
-    }
+    vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId);
   }
 
   streamMetaWUnLock(pMeta);
diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index fbf902a237..48a9d07a5d 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -141,7 +141,7 @@ SListNode* streamBackendAddCompare(void* backend, void* arg);
 void       streamBackendDelCompare(void* backend, void* arg);
 int32_t    streamStateCvtDataFormat(char* path, char* key, void* cfInst);
 
-STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkptId);
+STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId);
 void            taskDbDestroy(void* pBackend, bool flush);
 void            taskDbDestroy2(void* pBackend);
 int32_t         taskDbDoCheckpoint(void* arg, int64_t chkpId);
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index da8a24e6da..0ac10fe9fe 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -164,7 +164,7 @@ typedef enum ECHECKPOINT_BACKUP_TYPE {
 
 ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType();
 
-int32_t streamTaskDownloadCheckpointData(char* id, char* path);
+int32_t streamTaskDownloadCheckpointData(const char* id, char* path);
 int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
 int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask);
 
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index dd897dc431..ad5d759b0b 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -40,7 +40,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t
 
 void            destroyRocksdbCfInst(RocksdbCfInst* inst);
 int32_t         getCfIdx(const char* cfName);
-STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath);
+STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath);
 
 static int32_t backendCopyFiles(const char* src, const char* dst);
 
@@ -325,7 +325,7 @@ int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) {
   return complete == 1 ? 0 : -1;
 }
 
-int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) {
+int32_t rebuildFromRemoteChkp_rsync(const char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) {
   int32_t code = 0;
   if (taosIsDir(chkptPath)) {
     taosRemoveDir(chkptPath);
@@ -335,7 +335,7 @@ int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkptPath, int64_t checkpoi
   if (taosIsDir(defaultPath)) {
     taosRemoveDir(defaultPath);
     taosMulMkDir(defaultPath);
-    stDebug("clear local backend dir:%s succ", defaultPath);
+    stDebug("clear local default dir before download checkpoint data:%s succ", defaultPath);
   }
 
   code = streamTaskDownloadCheckpointData(key, chkptPath);
@@ -348,7 +348,7 @@ int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkptPath, int64_t checkpoi
   return backendCopyFiles(chkptPath, defaultPath);
 }
 
-int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
+int32_t rebuildFromRemoteChkp_s3(const char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
   int32_t code = streamTaskDownloadCheckpointData(key, chkpPath);
   if (code != 0) {
     return code;
@@ -383,14 +383,14 @@ int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char
   return code;
 }
 
-int32_t rebuildFromRemoteCheckpoint(char* key, char* chkpPath, int64_t checkpointId, char* defaultPath) {
+int32_t rebuildFromRemoteCheckpoint(const char* key, char* chkpPath, int64_t checkpointId, char* defaultPath) {
   ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
   if (type == DATA_UPLOAD_S3) {
     return rebuildFromRemoteChkp_s3(key, chkpPath, checkpointId, defaultPath);
   } else if (type == DATA_UPLOAD_RSYNC) {
     return rebuildFromRemoteChkp_rsync(key, chkpPath, checkpointId, defaultPath);
   } else {
-    stError("%s not remote backup checkpoint data for:%"PRId64, key, checkpointId);
+    stError("%s not remote backup checkpoint data for:%" PRId64" restore ", key, checkpointId);
   }
 
   return -1;
@@ -484,24 +484,26 @@ int32_t backendCopyFiles(const char* src, const char* dst) {
   return backendFileCopyFilesImpl(src, dst);
 }
 
-static int32_t rebuildFromLocalCheckpoint(char* pTaskIdStr, const char* checkpointPath, int64_t chkptId, const char* defaultPath) {
+static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* checkpointPath, int64_t checkpointId,
+                                          const char* defaultPath) {
   int32_t code = 0;
 
   if (taosIsDir(defaultPath)) {
     taosRemoveDir(defaultPath);
     taosMkDir(defaultPath);
-    stInfo("clear local backend dir:%s, done", defaultPath);
+    stInfo("%s clear local backend dir:%s, succ", pTaskIdStr, defaultPath);
   }
 
   if (taosIsDir(checkpointPath) && isValidCheckpoint(checkpointPath)) {
-    code = backendCopyFiles(checkpointPath, defaultPath);
+    stDebug("%s local checkpoint data existed, checkpointId:%d copy to backend dir", pTaskIdStr, checkpointId);
 
+    code = backendCopyFiles(checkpointPath, defaultPath);
     if (code != TSDB_CODE_SUCCESS) {
       taosRemoveDir(defaultPath);
       taosMkDir(defaultPath);
 
-      stError("%s failed to restart stream backend from %s, reason: %s, start to restart from empty path: %s",
-              pTaskIdStr, checkpointPath, tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath);
+      stError("%s failed to start stream backend from local %s, reason:%s, try download checkpoint from remote",
+              pTaskIdStr, checkpointPath, tstrerror(TAOS_SYSTEM_ERROR(errno)));
       code = TSDB_CODE_SUCCESS;
     } else {
       stInfo("%s copy checkpoint data from:%s to:%s succ, try to start stream backend", pTaskIdStr, checkpointPath,
@@ -509,18 +511,13 @@ static int32_t rebuildFromLocalCheckpoint(char* pTaskIdStr, const char* checkpoi
     }
   } else {
     code = TSDB_CODE_FAILED;
-    stError("%s no valid checkpoint data for checkpointId:%" PRId64 " in %s", pTaskIdStr, chkptId, checkpointPath);
+    stError("%s no valid data for checkpointId:%" PRId64 " in %s", pTaskIdStr, checkpointId, checkpointPath);
   }
 
   return code;
 }
 
-int32_t rebuildFromlocalDefault(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
-  int32_t code = 0;
-  return code;
-}
-
-int32_t rebuildDirFormCheckpoint(const char* path, char* key, int64_t chkptId, char** dbPrefixPath, char** dbPath) {
+int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId, char** dbPrefixPath, char** dbPath) {
   int32_t code = 0;
 
   char* prefixPath = taosMemoryCalloc(1, strlen(path) + 128);
@@ -533,13 +530,23 @@ int32_t rebuildDirFormCheckpoint(const char* path, char* key, int64_t chkptId, c
 
   char* defaultPath = taosMemoryCalloc(1, strlen(path) + 256);
   sprintf(defaultPath, "%s%s%s", prefixPath, TD_DIRSEP, "state");
+
   if (!taosIsDir(defaultPath)) {
     taosMulMkDir(defaultPath);
   }
-  stDebug("local default dir:%s, checkpointId:%" PRId64 ", key:%s succ", defaultPath, chkptId, key);
+
+  char* checkpointRoot = taosMemoryCalloc(1, strlen(path) + 256);
+  sprintf(checkpointRoot, "%s%s%s", prefixPath, TD_DIRSEP, "checkpoints");
+
+  if (!taosIsDir(checkpointRoot)) {
+    taosMulMkDir(checkpointRoot);
+  }
+  taosMemoryFree(checkpointRoot);
+
+  stDebug("%s check local default:%s, checkpointId:%" PRId64 " succ", key, defaultPath, chkptId);
 
   char* chkptPath = taosMemoryCalloc(1, strlen(path) + 256);
-  if (chkptId != 0) {
+  if (chkptId > 0) {
     sprintf(chkptPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId);
 
     code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath);
@@ -550,16 +557,9 @@ int32_t rebuildDirFormCheckpoint(const char* path, char* key, int64_t chkptId, c
     if (code != 0) {
       stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath,
              tstrerror(code), defaultPath);
-      code = taosMkDir(defaultPath);
-    }
-  } else {
-    sprintf(chkptPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint",
-            (int64_t)-1);
-
-    code = rebuildFromLocalCheckpoint(key, chkptPath, -1, defaultPath);
-    if (code != 0) {
-      code = taosMkDir(defaultPath);
     }
+  } else {  // no valid checkpoint id
+    stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data", key);
   }
 
   taosMemoryFree(chkptPath);
@@ -646,7 +646,7 @@ void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) {
   if (cfs != NULL) {
     rocksdb_list_column_families_destroy(cfs, nCf);
   }
-  stDebug("succ to init stream backend at %s, backend:%p, vgId:%d", backendPath, pHandle, vgId);
+  stDebug("%s init stream backend at %s, backend:%p, vgId:%d", backendPath, pHandle, vgId);
   taosMemoryFreeClear(backendPath);
 
   return (void*)pHandle;
@@ -1933,6 +1933,7 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta
   *stateFullPath = statePath;
   return 0;
 }
+
 void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) {
   STaskDbWrapper* p = pTaskDb;
   taosThreadMutexLock(&p->mutex);
@@ -1940,7 +1941,7 @@ void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) {
   taosThreadMutexUnlock(&p->mutex);
 }
 
-STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) {
+STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) {
   char*  err = NULL;
   char** cfNames = NULL;
   size_t nCf = 0;
@@ -1955,7 +1956,7 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) {
 
   cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err);
   if (nCf == 0) {
-    stInfo("newly create db, need to restart");
+    stInfo("%s newly create db, need to restart", key);
     // pre create db
     pTaskDb->db = rocksdb_open(pTaskDb->pCfOpts[0], dbPath, &err);
     if (pTaskDb->db == NULL) goto _EXIT;
@@ -1980,21 +1981,21 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) {
     cfNames = NULL;
   }
 
-  stDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb);
+  stDebug("init s-task backend in:%s, backend:%p, %s", dbPath, pTaskDb, key);
   return pTaskDb;
-_EXIT:
 
+_EXIT:
   taskDbDestroy(pTaskDb, false);
   if (err) taosMemoryFree(err);
   if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf);
   return NULL;
 }
 
-STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkptId) {
+STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId) {
   char* statePath = NULL;
   char* dbPath = NULL;
 
-  if (rebuildDirFormCheckpoint(path, key, chkptId, &statePath, &dbPath) != 0) {
+  if (restoreCheckpointData(path, key, chkptId, &statePath, &dbPath) != 0) {
     return NULL;
   }
 
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 389eccd66d..d09e5bf477 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -239,6 +239,14 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin
 
   taosThreadMutexLock(&pTask->lock);
 
+  if (pReq->checkpointId <= pInfo->checkpointId) {
+    stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64
+            " no need to update the checkpoint info, updated checkpointId:%" PRId64 " checkpointVer:%" PRId64 " ignored",
+            id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer);
+    taosThreadMutexUnlock(&pTask->lock);
+    return TSDB_CODE_SUCCESS;
+  }
+
   SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
 
   stDebug("s-task:%s vgId:%d status:%s start to update the checkpoint info, checkpointId:%" PRId64 "->%" PRId64
@@ -246,7 +254,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin
           id, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer,
           pInfo->checkpointTime, pReq->checkpointTs);
 
-  // in the
   if (pStatus->state != TASK_STATUS__DROPPING) {
     ASSERT(pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer);
 
@@ -593,7 +600,7 @@ int32_t downloadCheckpointDataByName(const char* id, const char* fname, const ch
   return 0;
 }
 
-int32_t streamTaskDownloadCheckpointData(char* id, char* path) {
+int32_t streamTaskDownloadCheckpointData(const char* id, char* path) {
   if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) {
     stError("down checkpoint data parameters invalid");
     return -1;
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 95fd057929..288d2eeaba 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -239,9 +239,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) {
   return 0;
 }
 
-int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) {
-  SStreamTask* pTask = arg;
-
+int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) {
   int64_t chkpId = pTask->chkInfo.checkpointId;
 
   taosThreadMutexLock(&pMeta->backendMutex);
@@ -1358,7 +1356,7 @@ static int32_t prepareBeforeStartTasks(SStreamMeta* pMeta, SArray** pList, int64
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
+int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expandFn) {
   int32_t code = TSDB_CODE_SUCCESS;
   int32_t vgId = pMeta->vgId;
   int64_t now = taosGetTimestampMs();
@@ -1392,8 +1390,17 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
       continue;
     }
 
-    // fill-history task can only be launched by related stream tasks.
     STaskExecStatisInfo* pInfo = &pTask->execInfo;
+
+    code = expandFn(pTask);
+    if (code != TSDB_CODE_SUCCESS) {
+      stError("s-task:0x%x vgId:%d failed to build stream backend", pTaskId->taskId, vgId);
+      streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
+      streamMetaReleaseTask(pMeta, pTask);
+      continue;
+    }
+
+    // fill-history task can only be launched by related stream tasks.
     if (pTask->info.fillHistory == 1) {
       stDebug("s-task:%s fill-history task wait related stream task start", pTask->id.idStr);
       streamMetaReleaseTask(pMeta, pTask);
@@ -1481,9 +1488,9 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta) {
   return true;
 }
 
-int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) {
+int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, __stream_task_expand_fn expandFn) {
   int32_t vgId = pMeta->vgId;
-  stInfo("vgId:%d start to task:0x%x by checking downstream status", vgId, taskId);
+  stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId);
 
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId);
   if (pTask == NULL) {
@@ -1501,6 +1508,10 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
 
   ASSERT(pTask->status.downstreamReady == 0);
 
+  if (pTask->pBackend == NULL) {  // todo handle the error code
+    int32_t code = expandFn(pTask);
+  }
+
   int32_t ret = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT);
   if (ret != TSDB_CODE_SUCCESS) {
     stError("s-task:%s vgId:%d failed to handle event:%d", pTask->id.idStr, pMeta->vgId, TASK_EVENT_INIT);
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 52002b7ea8..47324bd8c9 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -98,7 +98,7 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
   return winKeyCmprImpl(&pWin1->key, &pWin2->key);
 }
 
-SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) {
+SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) {
   SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
   stDebug("open stream state %p, %s", pState, path);
   if (pState == NULL) {
@@ -127,7 +127,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT);
 
   pState->parNameMap = tSimpleHashInit(1024, hashFn);
-  stInfo("succ to open state %p on backend %p 0x%" PRIx64 "-%d", pState, pMeta->streamBackend, pState->streamId,
+  stInfo("open state %p on backend %p 0x%" PRIx64 "-%d succ", pState, pMeta->streamBackend, pState->streamId,
          pState->taskId);
   return pState;
 

From 4fa8cfbffe9a22a7a1a76d8aeab53ae1caa5f743 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 14 May 2024 15:42:37 +0800
Subject: [PATCH 010/132] fix(stream): fix syntax error.

---
 source/libs/stream/src/streamBackendRocksdb.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index ad5d759b0b..2638e8db2f 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -580,11 +580,12 @@ bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId) {
   taosMemoryFree(state);
   return exist;
 }
+
 void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) {
   char*   backendPath = NULL;
   int32_t code = rebuildDirFromCheckpoint(streamPath, chkpId, &backendPath);
 
-  stDebug("start to init stream backend at %s, checkpointid: %" PRId64 " vgId:%d", backendPath, chkpId, vgId);
+  stDebug("start to init stream backend:%s, checkpointId:%" PRId64 " vgId:%d", backendPath, chkpId, vgId);
 
   uint32_t         dbMemLimit = nextPow2(tsMaxStreamBackendCache) << 20;
   SBackendWrapper* pHandle = taosMemoryCalloc(1, sizeof(SBackendWrapper));
@@ -643,10 +644,12 @@ void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) {
       goto _EXIT;
     }
   }
+
   if (cfs != NULL) {
     rocksdb_list_column_families_destroy(cfs, nCf);
   }
-  stDebug("%s init stream backend at %s, backend:%p, vgId:%d", backendPath, pHandle, vgId);
+
+  stDebug("init stream backend at %s, backend:%p, vgId:%d", backendPath, pHandle, vgId);
   taosMemoryFreeClear(backendPath);
 
   return (void*)pHandle;

From 1ccf743215b5e033175847c7919bcb11679828f8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 14 May 2024 16:19:21 +0800
Subject: [PATCH 011/132] fix(stream): fix syntax error.

---
 source/libs/stream/src/streamBackendRocksdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 2638e8db2f..c6e580dce9 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -495,7 +495,7 @@ static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* ch
   }
 
   if (taosIsDir(checkpointPath) && isValidCheckpoint(checkpointPath)) {
-    stDebug("%s local checkpoint data existed, checkpointId:%d copy to backend dir", pTaskIdStr, checkpointId);
+    stDebug("%s local checkpoint data existed, checkpointId:%" PRId64 " copy to backend dir", pTaskIdStr, checkpointId);
 
     code = backendCopyFiles(checkpointPath, defaultPath);
     if (code != TSDB_CODE_SUCCESS) {

From 4eacf86843f06055ac47a87a12a73768396e7195 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 14 May 2024 18:05:13 +0800
Subject: [PATCH 012/132] fix(tsdb): fix deadlock when stopping reader.

---
 source/libs/executor/src/executor.c | 17 ++++++++++-------
 source/libs/stream/src/streamMeta.c |  2 ++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 29f667cb66..2f360044c9 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -824,14 +824,17 @@ int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) {
   qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo));
   setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED);
 
-  taosWLockLatch(&pTaskInfo->lock);
-  while (qTaskIsExecuting(pTaskInfo)) {
-    taosMsleep(10);
+  while(1) {
+    taosWLockLatch(&pTaskInfo->lock);
+    if (qTaskIsExecuting(pTaskInfo)) { // let's wait for 100 ms and try again
+      taosWUnLockLatch(&pTaskInfo->lock);
+      taosMsleep(100);
+    } else { // not running now
+      pTaskInfo->code = rspCode;
+      taosWUnLockLatch(&pTaskInfo->lock);
+      return TSDB_CODE_SUCCESS;
+    }
   }
-  pTaskInfo->code = rspCode;
-  taosWUnLockLatch(&pTaskInfo->lock);
-
-  return TSDB_CODE_SUCCESS;
 }
 
 bool qTaskIsExecuting(qTaskInfo_t qinfo) {
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 288d2eeaba..ec5cf2f4f6 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1645,6 +1645,8 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
       streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false);
     }
   } else {
+    streamMetaRUnLock(pMeta);
+
     stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
             streamId, taskId, pMeta->vgId);
     code = TSDB_CODE_STREAM_TASK_NOT_EXIST;

From 610aa80e654cfd597f094840df9c60d38d05ed73 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 15 May 2024 00:08:38 +0800
Subject: [PATCH 013/132] fix(stream): init backend for fill-history task.

---
 include/libs/stream/tstream.h               |  1 -
 source/dnode/vnode/src/tqCommon/tqCommon.c  |  4 --
 source/libs/stream/inc/streamInt.h          |  2 +-
 source/libs/stream/src/streamMeta.c         | 48 +++++++++++++++------
 source/libs/stream/src/streamStartHistory.c |  2 +-
 source/libs/stream/src/streamTaskSm.c       |  2 +-
 6 files changed, 37 insertions(+), 22 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 1f5aa46f49..7cd8391c80 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -60,7 +60,6 @@ extern "C" {
 #define STREAM_EXEC_T_ADD_FAILED_TASK   (-7)
 // the load and start stream task should be executed after snode has started successfully, since the load of stream
 // tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources.
-#define STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS (-8)
 
 typedef struct SStreamTask      SStreamTask;
 typedef struct SStreamQueue     SStreamQueue;
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index dbda3a4541..544e820695 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -745,10 +745,6 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
   } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
     int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
     return code;
-  } else if (type == STREAM_EXEC_T_LOAD_AND_START_ALL_TASKS) {
-    streamMetaLoadAllTasks(pMeta);
-    int32_t code = streamMetaStartAllTasks(pMeta, tqExpandStreamTask);
-    return code;
   } else if (type == STREAM_EXEC_T_RESUME_TASK) {  // task resume to run after idle for a while
     SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
 
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 0ac10fe9fe..10bdccdb29 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -166,7 +166,7 @@ ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType();
 
 int32_t streamTaskDownloadCheckpointData(const char* id, char* path);
 int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
-int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask);
+int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask);
 
 typedef int32_t (*__stream_async_exec_fn_t)(void* param);
 
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index ec5cf2f4f6..5402c066a2 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1376,13 +1376,11 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
     return TSDB_CODE_SUCCESS;
   }
 
-  numOfTasks = taosArrayGetSize(pTaskList);
-
   // broadcast the check downstream tasks msg
+  numOfTasks = taosArrayGetSize(pTaskList);
   for (int32_t i = 0; i < numOfTasks; ++i) {
     SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
 
-    // todo: use hashTable instead
     SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId);
     if (pTask == NULL) {
       stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId);
@@ -1391,13 +1389,14 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
     }
 
     STaskExecStatisInfo* pInfo = &pTask->execInfo;
-
-    code = expandFn(pTask);
-    if (code != TSDB_CODE_SUCCESS) {
-      stError("s-task:0x%x vgId:%d failed to build stream backend", pTaskId->taskId, vgId);
-      streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
-      streamMetaReleaseTask(pMeta, pTask);
-      continue;
+    if (pTask->pBackend == NULL) {
+      code = expandFn(pTask);
+      if (code != TSDB_CODE_SUCCESS) {
+        stError("s-task:0x%x vgId:%d failed to expand stream backend", pTaskId->taskId, vgId);
+        streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
+        streamMetaReleaseTask(pMeta, pTask);
+        continue;
+      }
     }
 
     // fill-history task can only be launched by related stream tasks.
@@ -1407,6 +1406,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
       continue;
     }
 
+    // ready now, start the related fill-history task
     if (pTask->status.downstreamReady == 1) {
       if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
         stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task",
@@ -1429,7 +1429,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
     streamMetaReleaseTask(pMeta, pTask);
   }
 
-  stInfo("vgId:%d start tasks completed", pMeta->vgId);
+  stInfo("vgId:%d start all task(s) completed", pMeta->vgId);
   taosArrayDestroy(pTaskList);
   return code;
 }
@@ -1494,7 +1494,7 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
 
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId);
   if (pTask == NULL) {
-    stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, taskId);
+    stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId);
     streamMetaAddFailedTask(pMeta, streamId, taskId);
     return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
   }
@@ -1507,9 +1507,29 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
   }
 
   ASSERT(pTask->status.downstreamReady == 0);
-
-  if (pTask->pBackend == NULL) {  // todo handle the error code
+  if (pTask->pBackend == NULL) {
     int32_t code = expandFn(pTask);
+    if (code != TSDB_CODE_SUCCESS) {
+      streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
+      streamMetaReleaseTask(pMeta, pTask);
+      return code;
+    }
+
+    if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
+      SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId);
+      if (pHTask != NULL) {
+        if (pHTask->pBackend == NULL) {
+          code = expandFn(pHTask);
+          if (code != TSDB_CODE_SUCCESS) {
+            streamMetaAddFailedTaskSelf(pHTask, pInfo->readyTs);
+            streamMetaReleaseTask(pMeta, pHTask);
+            return code;
+          }
+        }
+
+        streamMetaReleaseTask(pMeta, pHTask);
+      }
+    }
   }
 
   int32_t ret = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT);
diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c
index 6882f6617d..7a864a60d2 100644
--- a/source/libs/stream/src/streamStartHistory.c
+++ b/source/libs/stream/src/streamStartHistory.c
@@ -155,7 +155,7 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) {
+int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) {
   // set the state to be ready
   streamTaskSetReady(pTask);
   streamTaskSetRangeStreamCalc(pTask);
diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index cced6a6b84..82ea2f88ef 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -584,7 +584,7 @@ void doInitStateTransferTable(void) {
   // initialization event handle
   STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, NULL);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanHistoryTaskReady, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // scan-history related event

From 042ed3caff99aaeba336273840aba2370c0bcf0d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 15 May 2024 10:08:56 +0800
Subject: [PATCH 014/132] fix(stream): fix invalid unlock.

---
 source/libs/stream/src/streamMeta.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 5402c066a2..39ed23730e 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -257,19 +257,18 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
     return 0;
   }
 
-  STaskDbWrapper* pBackend = taskDbOpen(pMeta->path, key, chkpId);
+  STaskDbWrapper* pBackend = NULL;
   while (1) {
-    if (pBackend == NULL) {
-      taosThreadMutexUnlock(&pMeta->backendMutex);
-      taosMsleep(1000);
-      stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key);
-    } else {
-      taosThreadMutexUnlock(&pMeta->backendMutex);
+    pBackend = taskDbOpen(pMeta->path, key, chkpId);
+    if (pBackend != NULL) {
       break;
     }
 
+    taosThreadMutexUnlock(&pMeta->backendMutex);
+    taosMsleep(1000);
+
+    stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key);
     taosThreadMutexLock(&pMeta->backendMutex);
-    pBackend = taskDbOpen(pMeta->path, key, chkpId);
   }
 
   int64_t tref = taosAddRef(taskDbWrapperId, pBackend);

From abd90c733a526904982fd95dfc72f631b39481e0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 15 May 2024 14:24:35 +0800
Subject: [PATCH 015/132] fix(stream): remove backend files only after the
 in-memory task is dropped.

---
 include/libs/stream/tstream.h                 |  2 ++
 source/dnode/vnode/src/tqCommon/tqCommon.c    | 15 ++--------
 source/libs/stream/src/streamBackendRocksdb.c |  6 +++-
 source/libs/stream/src/streamMeta.c           |  2 ++
 source/libs/stream/src/streamTask.c           | 28 ++++++++++++++-----
 tests/system-test/1-insert/drop.py            |  1 +
 6 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 7cd8391c80..0840694964 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -290,6 +290,7 @@ typedef struct SStreamStatus {
   int64_t        lastExecTs;     // last exec time stamp
   int32_t        inScanHistorySentinel;
   bool           appendTranstateBlock;  // has append the transfer state data block already
+  bool           removeBackendFiles;    // remove backend files on disk when free stream tasks
 } SStreamStatus;
 
 typedef struct SDataRange {
@@ -675,6 +676,7 @@ void    streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
 void    streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
 int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key);
 bool    streamTaskIsSinkTask(const SStreamTask* pTask);
+void    streamTaskSetRemoveBackendFiles(SStreamTask* pTask);
 
 void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
 void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 544e820695..b5b5ef8755 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -584,18 +584,6 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
   return code;
 }
 
-static void tqStreamRemoveTaskBackend(SStreamMeta* pMeta, const STaskId* pId) {
-  char taskKey[128] = {0};
-  sprintf(taskKey, "0x%" PRIx64 "-0x%x", pId->streamId, (int32_t)pId->taskId);
-
-  char* path = taosMemoryCalloc(1, strlen(pMeta->path) + 128);
-  sprintf(path, "%s%s%s", pMeta->path, TD_DIRSEP, taskKey);
-  taosRemoveDir(path);
-
-  tqInfo("vgId:%d drop stream task:0x%x file:%s", pMeta->vgId, (int32_t)pId->taskId, path);
-  taosMemoryFree(path);
-}
-
 int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
   SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
 
@@ -616,6 +604,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
       hTaskId.taskId = pTask->hTaskInfo.id.taskId;
     }
 
+    streamTaskSetRemoveBackendFiles(pTask);
     streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
     streamMetaReleaseTask(pMeta, pTask);
   }
@@ -642,7 +631,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
 
   streamMetaWUnLock(pMeta);
 
-  tqStreamRemoveTaskBackend(pMeta, &id);
+//  tqStreamRemoveTaskBackend(pMeta, &id);
   return 0;
 }
 
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index c6e580dce9..b6bbe2e383 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -1968,11 +1968,15 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) {
     if (cfNames != NULL) {
       rocksdb_list_column_families_destroy(cfNames, nCf);
     }
+
     taosMemoryFree(err);
     err = NULL;
 
     cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err);
-    ASSERT(err == NULL);
+    if (err != NULL) {
+      stError("%s failed to create column-family, %s, %d, reason:%s", key, dbPath, nCf, err);
+      goto _EXIT;
+    }
   }
 
   if (taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf) != 0) {
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 39ed23730e..27bd001588 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1521,7 +1521,9 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
           code = expandFn(pHTask);
           if (code != TSDB_CODE_SUCCESS) {
             streamMetaAddFailedTaskSelf(pHTask, pInfo->readyTs);
+
             streamMetaReleaseTask(pMeta, pHTask);
+            streamMetaReleaseTask(pMeta, pTask);
             return code;
           }
         }
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 72302f981d..2cb388954d 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -187,8 +187,9 @@ int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId) {
 }
 
 void tFreeStreamTask(SStreamTask* pTask) {
-  char*                p = NULL;
-  int32_t              taskId = pTask->id.taskId;
+  char*   p = NULL;
+  int32_t taskId = pTask->id.taskId;
+
   STaskExecStatisInfo* pStatis = &pTask->execInfo;
 
   ETaskStatus status1 = TASK_STATUS__UNINIT;
@@ -200,7 +201,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
   }
   taosThreadMutexUnlock(&pTask->lock);
 
-  stDebug("start to free s-task:0x%x, %p, state:%s", taskId, pTask, p);
+  stDebug("start to free s-task:0x%x %p, state:%s", taskId, pTask, p);
 
   SCheckpointInfo* pCkInfo = &pTask->chkInfo;
   stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64
@@ -275,10 +276,6 @@ void tFreeStreamTask(SStreamTask* pTask) {
     taskDbRemoveRef(pTask->pBackend);
   }
 
-  if (pTask->id.idStr != NULL) {
-    taosMemoryFree((void*)pTask->id.idStr);
-  }
-
   if (pTask->pNameMap) {
     tSimpleHashCleanup(pTask->pNameMap);
   }
@@ -292,6 +289,19 @@ void tFreeStreamTask(SStreamTask* pTask) {
 
   pTask->outputInfo.pNodeEpsetUpdateList = taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList);
 
+  if ((pTask->status.removeBackendFiles) && (pTask->pMeta != NULL)) {
+    char* path = taosMemoryCalloc(1, strlen(pTask->pMeta->path) + 128);
+    sprintf(path, "%s%s%s", pTask->pMeta->path, TD_DIRSEP, pTask->id.idStr);
+    taosRemoveDir(path);
+
+    stInfo("s-task:0x%x vgId:%d remove all backend files:%s", taskId, pTask->pMeta->vgId, path);
+    taosMemoryFree(path);
+  }
+
+  if (pTask->id.idStr != NULL) {
+    taosMemoryFree((void*)pTask->id.idStr);
+  }
+
   taosMemoryFree(pTask);
   stDebug("s-task:0x%x free task completed", taskId);
 }
@@ -896,4 +906,8 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) {
     return code;
   }
   return streamTrySchedExec(pTask);
+}
+
+void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) {
+  pTask->status.removeBackendFiles = true;
 }
\ No newline at end of file
diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py
index 21817ef20d..493e1491b8 100644
--- a/tests/system-test/1-insert/drop.py
+++ b/tests/system-test/1-insert/drop.py
@@ -20,6 +20,7 @@ from util.common import *
 from util.sqlset import *
 
 class TDTestCase:
+    updatecfgDict = {'stdebugflag':143}
     def init(self, conn, logSql, replicaVar=1):
         self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)

From 2b442119288c263bef2d5299512448e088d4e3c4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 15 May 2024 14:52:59 +0800
Subject: [PATCH 016/132] fix(test): update test case and fix a typo.

---
 source/libs/stream/src/streamBackendRocksdb.c | 2 +-
 source/util/test/cfgTest.cpp                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index b6bbe2e383..398980dfba 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -1974,7 +1974,7 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) {
 
     cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err);
     if (err != NULL) {
-      stError("%s failed to create column-family, %s, %d, reason:%s", key, dbPath, nCf, err);
+      stError("%s failed to create column-family, %s, %" PRIzu ", reason:%s", key, dbPath, nCf, err);
       goto _EXIT;
     }
   }
diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp
index 92422b6a80..9f8645b14c 100644
--- a/source/util/test/cfgTest.cpp
+++ b/source/util/test/cfgTest.cpp
@@ -67,7 +67,7 @@ TEST_F(CfgTest, 02_Basic) {
 
   SConfigItem* pItem = NULL;
   SConfigIter* pIter = cfgCreateIter(pConfig);
-  while((pItem == cfgNextIter(pIter)) != NULL) {
+  while((pItem = cfgNextIter(pIter)) != NULL) {
     switch (pItem->dtype) {
       case CFG_DTYPE_BOOL:
         printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval);

From 130f7a292d718aba1c62db4f98cb81ca338f39bf Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 19 May 2024 22:36:09 +0800
Subject: [PATCH 017/132] fix(stream): reset the error code when trying to
 restore checkpoint.

---
 source/libs/stream/src/streamBackendRocksdb.c |  5 +++--
 source/libs/stream/src/streamMeta.c           | 10 +++++-----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 398980dfba..5b03876f02 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -390,7 +390,7 @@ int32_t rebuildFromRemoteCheckpoint(const char* key, char* chkpPath, int64_t che
   } else if (type == DATA_UPLOAD_RSYNC) {
     return rebuildFromRemoteChkp_rsync(key, chkpPath, checkpointId, defaultPath);
   } else {
-    stError("%s not remote backup checkpoint data for:%" PRId64" restore ", key, checkpointId);
+    stError("%s no remote backup checkpoint data for:%" PRId64, key, checkpointId);
   }
 
   return -1;
@@ -543,7 +543,7 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
   }
   taosMemoryFree(checkpointRoot);
 
-  stDebug("%s check local default:%s, checkpointId:%" PRId64 " succ", key, defaultPath, chkptId);
+  stDebug("%s check local backend dir:%s, checkpointId:%" PRId64 " succ", key, defaultPath, chkptId);
 
   char* chkptPath = taosMemoryCalloc(1, strlen(path) + 256);
   if (chkptId > 0) {
@@ -557,6 +557,7 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
     if (code != 0) {
       stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath,
              tstrerror(code), defaultPath);
+      code = 0;    // reset the error code
     }
   } else {  // no valid checkpoint id
     stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data", key);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 27bd001588..a6453871c8 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -791,11 +791,11 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) {
 }
 
 int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) {
-  int64_t chkpId = 0;
+  int64_t checkpointId = 0;
 
   TBC* pCur = NULL;
   if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) {
-    return chkpId;
+    return checkpointId;
   }
 
   void*    pKey = NULL;
@@ -816,16 +816,16 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) {
     }
     tDecoderClear(&decoder);
 
-    chkpId = TMAX(chkpId, info.checkpointId);
+    checkpointId = TMAX(checkpointId, info.checkpointId);
   }
 
-  stDebug("get max chkp id: %" PRId64 "", chkpId);
+  stDebug("vgId:%d get max checkpointId:%" PRId64, pMeta->vgId, checkpointId);
 
   tdbFree(pKey);
   tdbFree(pVal);
   tdbTbcClose(pCur);
 
-  return chkpId;
+  return checkpointId;
 }
 
 // not allowed to return error code

From f6af165d9ac40a6142ca6c740fb3e6ae20124928 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 21 May 2024 10:09:55 +0800
Subject: [PATCH 018/132] refactor(stream): add more info in info-meta

---
 source/common/src/systable.c            |  3 +++
 source/dnode/mnode/impl/src/mndStream.c | 12 ++++++++++++
 source/dnode/vnode/src/tq/tqSink.c      |  4 ++--
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/source/common/src/systable.c b/source/common/src/systable.c
index 9de682dd3a..d5dc1581a4 100644
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@@ -187,6 +187,8 @@ static const SSysDbTableSchema streamTaskSchema[] = {
     {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "input_idle", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "dispatch_data", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "info", .bytes = 35, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
@@ -195,6 +197,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
     {.name = "checkpoint_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
     {.name = "checkpoint_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "checkpoint_version", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
+    {.name = "checkpoint_size", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "checkpoint_backup", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "ds_err_info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index fb15e4b857..03385ea226 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1629,6 +1629,14 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
   colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
 
+  // input idle
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetNULL(pColInfo, numOfRows);
+
+  // dispatch data
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetNULL(pColInfo, numOfRows);
+
   // output queue
   //          sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate);
   //        STR_TO_VARSTR(vbuf, buf);
@@ -1679,6 +1687,10 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
   colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestVer, false);
 
+  // checkpoint size
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetNULL(pColInfo, numOfRows);
+
   // checkpoint backup status
   pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
   colDataSetVal(pColInfo, numOfRows, 0, true);
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index c0f58fc3ec..ce6db55cef 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -744,7 +744,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
       tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid);
     }
   } else {
-    // The auto-create option will always set to be open for those submit messages, which arrive during the period
+    // The auto-create option will always set to be open for those submit messages, which arrives during the period
     // the creating of the destination table, due to the absence of the user-specified table in TSDB. When scanning
     // data from WAL, those submit messages, with auto-created table option, will be discarded expect the first, for
     // those mismatched table uids. Only the FIRST table has the correct table uid, and those remain all have
@@ -752,7 +752,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
     SMetaReader mr = {0};
     metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK);
 
-    // table not in cache, let's try the extract it from tsdb meta
+    // table not in cache, let's try to extract it from tsdb meta
     if (metaGetTableEntryByName(&mr, dstTableName) < 0) {
       metaReaderClear(&mr);
 

From be39044b669dd928730f677923ea6ae19a4781d6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 11:04:54 +0800
Subject: [PATCH 019/132] fix(stream): add more info

---
 include/libs/stream/tstream.h           | 18 ++++--
 source/common/src/systable.c            | 12 ++--
 source/dnode/mnode/impl/src/mndStream.c | 75 +++++++++++++++++++++----
 source/libs/stream/src/streamExec.c     | 31 ++++++++--
 source/libs/stream/src/streamMeta.c     |  7 ++-
 source/libs/stream/src/streamTask.c     |  4 ++
 source/libs/stream/src/streammsg.c      | 12 ++++
 7 files changed, 136 insertions(+), 23 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 0840694964..ea0f1824b3 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -357,8 +357,12 @@ typedef struct STaskExecStatisInfo {
   double        step2El;
   int32_t       updateCount;
   int64_t       latestUpdateTs;
-  int32_t       processDataBlocks;
-  int64_t       processDataSize;
+  int32_t       inputDataBlocks;
+  int64_t       inputDataSize;
+  double        procsThroughput;
+  int64_t       outputDataBlocks;
+  int64_t       outputDataSize;
+  double        outputThroughput;
   int32_t       dispatch;
   int64_t       dispatchDataSize;
   int32_t       checkpoint;
@@ -566,6 +570,8 @@ typedef struct STaskCkptInfo {
   int64_t latestId;            // saved checkpoint id
   int64_t latestVer;           // saved checkpoint ver
   int64_t latestTime;          // latest checkpoint time
+  int64_t latestSize;          // latest checkpoint size
+  int8_t  remoteBackup;        // latest checkpoint backup done
   int64_t activeId;  // current active checkpoint id
   int32_t activeTransId;       // checkpoint trans id
   int8_t  failed;              // denote if the checkpoint is failed or not
@@ -583,8 +589,12 @@ typedef struct STaskStatusEntry {
   int64_t       inputQUnchangeCounter;
   double        inputQUsed;  // in MiB
   double        inputRate;
-  double        sinkQuota;     // existed quota size for sink task
-  double        sinkDataSize;  // sink to dst data size
+  double        procsThroughput;     // duration between one element put into input queue and being processed.
+  double        procsTotal;          // duration between one element put into input queue and being processed.
+  double        outputThroughput;    // the size of dispatched result blocks in bytes
+  double        outputTotal;         // the size of dispatched result blocks in bytes
+  double        sinkQuota;           // existed quota size for sink task
+  double        sinkDataSize;        // sink to dst data size
   int64_t       startTime;
   int64_t       startCheckpointId;
   int64_t       startCheckpointVer;
diff --git a/source/common/src/systable.c b/source/common/src/systable.c
index d5dc1581a4..297f43399f 100644
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@@ -187,8 +187,12 @@ static const SSysDbTableSchema streamTaskSchema[] = {
     {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "input_idle", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "dispatch_data", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "process_total", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "process_throughput", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "out_total", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "out_throughput", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+//    {.name = "dispatch_throughput", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+//    {.name = "dispatch_total", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "info", .bytes = 35, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
@@ -196,10 +200,10 @@ static const SSysDbTableSchema streamTaskSchema[] = {
     {.name = "start_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "checkpoint_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
     {.name = "checkpoint_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
-    {.name = "checkpoint_version", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
+    {.name = "checkpoint_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "checkpoint_size", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "checkpoint_backup", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "ds_err_info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "extra_info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 };
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 03385ea226..105565067c 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1620,22 +1620,75 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false);
 
   // input queue
-  char        vbuf[30] = {0};
-  char        buf[25] = {0};
-  const char *queueInfoStr = "%4.2fMiB (%5.2f%)";
+  char        vbuf[40] = {0};
+  char        buf[40] = {0};
+  const char *queueInfoStr = "%4.2f MiB (%6.2f%)";
   sprintf(buf, queueInfoStr, pe->inputQUsed, pe->inputRate);
   STR_TO_VARSTR(vbuf, buf);
 
   pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
   colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
 
-  // input idle
-  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataSetNULL(pColInfo, numOfRows);
+  // input total
+  const char* formatTotalMb = "%7.2f MiB";
+  const char* formatTotalGb = "%7.2f GiB";
+  if (pe->procsTotal < 1024) {
+    sprintf(buf, formatTotalMb, pe->procsTotal);
+  } else {
+    sprintf(buf, formatTotalGb, pe->procsTotal / 1024);
+  }
+
+  memset(vbuf, 0, tListLen(vbuf));
+  STR_TO_VARSTR(vbuf, buf);
 
-  // dispatch data
   pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-  colDataSetNULL(pColInfo, numOfRows);
+  colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
+
+  // process throughput
+  const char* formatKb = "%7.2f KiB/s";
+  const char* formatMb = "%7.2f MiB/s";
+  if (pe->procsThroughput < 1024) {
+    sprintf(buf, formatKb, pe->procsThroughput);
+  } else {
+    sprintf(buf, formatMb, pe->procsThroughput / 1024);
+  }
+
+  memset(vbuf, 0, tListLen(vbuf));
+  STR_TO_VARSTR(vbuf, buf);
+
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+  colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
+
+  // output total
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+
+  if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
+    colDataSetNULL(pColInfo, numOfRows);
+  } else {
+    sprintf(buf, formatTotalMb, pe->outputTotal);
+    memset(vbuf, 0, tListLen(vbuf));
+    STR_TO_VARSTR(vbuf, buf);
+
+    colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
+  }
+
+  // output throughput
+  pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+
+  if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
+    colDataSetNULL(pColInfo, numOfRows);
+  } else {
+    if (pe->outputThroughput < 1024) {
+      sprintf(buf, formatKb, pe->outputThroughput);
+    } else {
+      sprintf(buf, formatMb, pe->outputThroughput / 1024);
+    }
+
+    memset(vbuf, 0, tListLen(vbuf));
+    STR_TO_VARSTR(vbuf, buf);
+
+    colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
+  }
 
   // output queue
   //          sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate);
@@ -1646,12 +1699,14 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
 
   // info
   if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
-    const char *sinkStr = "%.2fMiB";
+    const char *sinkStr = "%.2f MiB";
     sprintf(buf, sinkStr, pe->sinkDataSize);
   } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     // offset info
     const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]";
-    sprintf(buf, offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer);
+    snprintf(buf, tListLen(buf), offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer);
+  } else {
+    memset(buf, 0, tListLen(buf));
   }
 
   STR_TO_VARSTR(vbuf, buf);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 9747ebd2ff..9daca9a99c 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -96,7 +96,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i
   *totalSize = 0;
 
   int32_t size = 0;
-  int32_t numOfBlocks = 0;
+  int32_t numOfBlocks= 0;
   SArray* pRes = NULL;
 
   while (1) {
@@ -586,6 +586,9 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
       }
     }
 
+    pTask->execInfo.inputDataBlocks += numOfBlocks;
+    pTask->execInfo.inputDataSize += blockSize;
+
     // dispatch checkpoint msg to all downstream tasks
     int32_t type = pInput->type;
     if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
@@ -601,11 +604,21 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
     if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
       ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT);
 
+      int64_t st = taosGetTimestampMs();
+
       // here only handle the data block sink operation
       if (type == STREAM_INPUT__DATA_BLOCK) {
         pTask->execInfo.sink.dataSize += blockSize;
         stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize));
         doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput);
+
+        double el = (taosGetTimestampMs() - st) / 1000.0;
+        if (fabs(el - 0.0) <= DBL_EPSILON) {
+          pTask->execInfo.procsThroughput = 0;
+        } else {
+          pTask->execInfo.procsThroughput = (blockSize / el);
+        }
+
         continue;
       }
     }
@@ -639,13 +652,23 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
     int64_t ver = pTask->chkInfo.processedVer;
     doSetStreamInputBlock(pTask, pInput, &ver, id);
 
-    int64_t resSize = 0;
+    int64_t totalSize = 0;
     int32_t totalBlocks = 0;
-    streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks);
+    streamTaskExecImpl(pTask, pInput, &totalSize, &totalBlocks);
 
     double el = (taosGetTimestampMs() - st) / 1000.0;
     stDebug("s-task:%s batch of input blocks exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", id, el,
-           SIZE_IN_MiB(resSize), totalBlocks);
+           SIZE_IN_MiB(totalSize), totalBlocks);
+
+    pTask->execInfo.outputDataBlocks += totalBlocks;
+    pTask->execInfo.outputDataSize += totalSize;
+    if (fabs(el - 0.0) <= DBL_EPSILON) {
+      pTask->execInfo.procsThroughput = 0;
+      pTask->execInfo.outputThroughput = 0;
+    } else {
+      pTask->execInfo.outputThroughput = (totalSize / el);
+      pTask->execInfo.procsThroughput = (blockSize / el);
+    }
 
     SCheckpointInfo* pInfo = &pTask->chkInfo;
 
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index a6453871c8..7f1128b929 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1017,8 +1017,13 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
         .checkpointInfo.latestId = (*pTask)->chkInfo.checkpointId,
         .checkpointInfo.latestVer = (*pTask)->chkInfo.checkpointVer,
         .checkpointInfo.latestTime = (*pTask)->chkInfo.checkpointTime,
+        .checkpointInfo.latestSize = 0,
+        .checkpointInfo.remoteBackup = 0,
         .hTaskId = (*pTask)->hTaskInfo.id.taskId,
-
+        .procsTotal = SIZE_IN_MiB((*pTask)->execInfo.inputDataSize),
+        .outputTotal = SIZE_IN_MiB((*pTask)->execInfo.outputDataSize),
+        .procsThroughput = SIZE_IN_KiB((*pTask)->execInfo.procsThroughput),
+        .outputThroughput = SIZE_IN_KiB((*pTask)->execInfo.outputThroughput),
         .startCheckpointId = (*pTask)->execInfo.startCheckpointId,
         .startCheckpointVer = (*pTask)->execInfo.startCheckpointVer,
     };
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 2cb388954d..c056e2a4b6 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -771,6 +771,10 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc)
   pDst->stage = pSrc->stage;
   pDst->inputQUsed = pSrc->inputQUsed;
   pDst->inputRate = pSrc->inputRate;
+  pDst->procsTotal = pSrc->procsTotal;
+  pDst->procsThroughput = pSrc->procsThroughput;
+  pDst->outputTotal = pSrc->outputTotal;
+  pDst->outputThroughput = pSrc->outputThroughput;
   pDst->processedVer = pSrc->processedVer;
   pDst->verRange = pSrc->verRange;
   pDst->sinkQuota = pSrc->sinkQuota;
diff --git a/source/libs/stream/src/streammsg.c b/source/libs/stream/src/streammsg.c
index 5e52b927c6..9b69833234 100644
--- a/source/libs/stream/src/streammsg.c
+++ b/source/libs/stream/src/streammsg.c
@@ -335,6 +335,10 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
     if (tEncodeI32(pEncoder, ps->nodeId) < 0) return -1;
     if (tEncodeDouble(pEncoder, ps->inputQUsed) < 0) return -1;
     if (tEncodeDouble(pEncoder, ps->inputRate) < 0) return -1;
+    if (tEncodeDouble(pEncoder, ps->procsTotal) < 0) return -1;
+    if (tEncodeDouble(pEncoder, ps->procsThroughput) < 0) return -1;
+    if (tEncodeDouble(pEncoder, ps->outputTotal) < 0) return -1;
+    if (tEncodeDouble(pEncoder, ps->outputThroughput) < 0) return -1;
     if (tEncodeDouble(pEncoder, ps->sinkQuota) < 0) return -1;
     if (tEncodeDouble(pEncoder, ps->sinkDataSize) < 0) return -1;
     if (tEncodeI64(pEncoder, ps->processedVer) < 0) return -1;
@@ -346,6 +350,8 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
     if (tEncodeI64(pEncoder, ps->checkpointInfo.latestId) < 0) return -1;
     if (tEncodeI64(pEncoder, ps->checkpointInfo.latestVer) < 0) return -1;
     if (tEncodeI64(pEncoder, ps->checkpointInfo.latestTime) < 0) return -1;
+    if (tEncodeI64(pEncoder, ps->checkpointInfo.latestSize) < 0) return -1;
+    if (tEncodeI8(pEncoder, ps->checkpointInfo.remoteBackup) < 0) return -1;
     if (tEncodeI64(pEncoder, ps->startTime) < 0) return -1;
     if (tEncodeI64(pEncoder, ps->startCheckpointId) < 0) return -1;
     if (tEncodeI64(pEncoder, ps->startCheckpointVer) < 0) return -1;
@@ -381,6 +387,10 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
     if (tDecodeI32(pDecoder, &entry.nodeId) < 0) return -1;
     if (tDecodeDouble(pDecoder, &entry.inputQUsed) < 0) return -1;
     if (tDecodeDouble(pDecoder, &entry.inputRate) < 0) return -1;
+    if (tDecodeDouble(pDecoder, &entry.procsTotal) < 0) return -1;
+    if (tDecodeDouble(pDecoder, &entry.procsThroughput) < 0) return -1;
+    if (tDecodeDouble(pDecoder, &entry.outputTotal) < 0) return -1;
+    if (tDecodeDouble(pDecoder, &entry.outputThroughput) < 0) return -1;
     if (tDecodeDouble(pDecoder, &entry.sinkQuota) < 0) return -1;
     if (tDecodeDouble(pDecoder, &entry.sinkDataSize) < 0) return -1;
     if (tDecodeI64(pDecoder, &entry.processedVer) < 0) return -1;
@@ -393,6 +403,8 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
     if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestId) < 0) return -1;
     if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestVer) < 0) return -1;
     if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestTime) < 0) return -1;
+    if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestSize) < 0) return -1;
+    if (tDecodeI8(pDecoder, &entry.checkpointInfo.remoteBackup) < 0) return -1;
     if (tDecodeI64(pDecoder, &entry.startTime) < 0) return -1;
     if (tDecodeI64(pDecoder, &entry.startCheckpointId) < 0) return -1;
     if (tDecodeI64(pDecoder, &entry.startCheckpointVer) < 0) return -1;

From 77852fa70249ccec4dc10ebee573e8bc9f191bf5 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 13:20:45 +0800
Subject: [PATCH 020/132] fix(stream): avoid data overflow and fix test cases.

---
 source/dnode/mnode/impl/src/mndStream.c          | 2 +-
 tests/system-test/0-others/information_schema.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 105565067c..d65d708bb8 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1700,7 +1700,7 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   // info
   if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
     const char *sinkStr = "%.2f MiB";
-    sprintf(buf, sinkStr, pe->sinkDataSize);
+    snprintf(buf, tListLen(buf), sinkStr, pe->sinkDataSize);
   } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     // offset info
     const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]";
diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py
index 944b2fbb1e..965cdee9ce 100644
--- a/tests/system-test/0-others/information_schema.py
+++ b/tests/system-test/0-others/information_schema.py
@@ -222,7 +222,7 @@ class TDTestCase:
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
         tdLog.info(len(tdSql.queryResult))
-        tdSql.checkEqual(True, len(tdSql.queryResult) in range(254, 255))
+        tdSql.checkEqual(True, len(tdSql.queryResult) in range(258, 259))
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
         tdSql.checkEqual(54, len(tdSql.queryResult))

From 565d1f408cdd8f0bd1ed2e7937aad0f4fda5d431 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 13:26:49 +0800
Subject: [PATCH 021/132] fix(stream): adjust buf size, and avoid overflow

---
 source/dnode/mnode/impl/src/mndStream.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index d65d708bb8..98f438bf88 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1620,10 +1620,10 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false);
 
   // input queue
-  char        vbuf[40] = {0};
-  char        buf[40] = {0};
+  char        vbuf[37] = {0};
+  char        buf[35] = {0};
   const char *queueInfoStr = "%4.2f MiB (%6.2f%)";
-  sprintf(buf, queueInfoStr, pe->inputQUsed, pe->inputRate);
+  snprintf(buf, tListLen(buf), queueInfoStr, pe->inputQUsed, pe->inputRate);
   STR_TO_VARSTR(vbuf, buf);
 
   pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@@ -1633,9 +1633,9 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   const char* formatTotalMb = "%7.2f MiB";
   const char* formatTotalGb = "%7.2f GiB";
   if (pe->procsTotal < 1024) {
-    sprintf(buf, formatTotalMb, pe->procsTotal);
+    snprintf(buf, tListLen(buf), formatTotalMb, pe->procsTotal);
   } else {
-    sprintf(buf, formatTotalGb, pe->procsTotal / 1024);
+    snprintf(buf, tListLen(buf), formatTotalGb, pe->procsTotal / 1024);
   }
 
   memset(vbuf, 0, tListLen(vbuf));
@@ -1648,9 +1648,9 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   const char* formatKb = "%7.2f KiB/s";
   const char* formatMb = "%7.2f MiB/s";
   if (pe->procsThroughput < 1024) {
-    sprintf(buf, formatKb, pe->procsThroughput);
+    snprintf(buf, tListLen(buf), formatKb, pe->procsThroughput);
   } else {
-    sprintf(buf, formatMb, pe->procsThroughput / 1024);
+    snprintf(buf, tListLen(buf), formatMb, pe->procsThroughput / 1024);
   }
 
   memset(vbuf, 0, tListLen(vbuf));
@@ -1679,9 +1679,9 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
     colDataSetNULL(pColInfo, numOfRows);
   } else {
     if (pe->outputThroughput < 1024) {
-      sprintf(buf, formatKb, pe->outputThroughput);
+      snprintf(buf, tListLen(buf), formatKb, pe->outputThroughput);
     } else {
-      sprintf(buf, formatMb, pe->outputThroughput / 1024);
+      snprintf(buf, tListLen(buf), formatMb, pe->outputThroughput / 1024);
     }
 
     memset(vbuf, 0, tListLen(vbuf));

From 317742acc1913e67c6e8f66a39bf398536ee231c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 13:29:38 +0800
Subject: [PATCH 022/132] fix(stream): adjust buf size.

---
 source/common/src/systable.c            | 2 +-
 source/dnode/mnode/impl/src/mndStream.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/common/src/systable.c b/source/common/src/systable.c
index 297f43399f..4fdbe1b9f0 100644
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@@ -194,7 +194,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
 //    {.name = "dispatch_throughput", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "dispatch_total", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "info", .bytes = 35, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "info", .bytes = 40, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
     {.name = "start_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "start_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 98f438bf88..cb0a6c5d99 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1620,8 +1620,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false);
 
   // input queue
-  char        vbuf[37] = {0};
-  char        buf[35] = {0};
+  char        vbuf[40] = {0};
+  char        buf[38] = {0};
   const char *queueInfoStr = "%4.2f MiB (%6.2f%)";
   snprintf(buf, tListLen(buf), queueInfoStr, pe->inputQUsed, pe->inputRate);
   STR_TO_VARSTR(vbuf, buf);

From d13aff3b7045f7d874fbe2ba4a4777576b019e79 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 13:33:17 +0800
Subject: [PATCH 023/132] fix(stream): adjust buf size, and avoid overflow

---
 source/common/src/systable.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/common/src/systable.c b/source/common/src/systable.c
index 4fdbe1b9f0..980b590748 100644
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@@ -194,16 +194,16 @@ static const SSysDbTableSchema streamTaskSchema[] = {
 //    {.name = "dispatch_throughput", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "dispatch_total", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "info", .bytes = 40, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "info", .bytes = 40+ VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
     {.name = "start_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "start_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "checkpoint_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
     {.name = "checkpoint_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "checkpoint_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
-    {.name = "checkpoint_size", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "checkpoint_backup", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "extra_info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "checkpoint_size", .bytes = 14 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "checkpoint_backup", .bytes = 14 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "extra_info", .bytes = 25 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 };

From 5aae69338c397e8457499f80cb56b088e3ab6f84 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 16:13:13 +0800
Subject: [PATCH 024/132] fix(test): update test cases.

---
 tests/system-test/0-others/information_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py
index 965cdee9ce..bf2f1c01f6 100644
--- a/tests/system-test/0-others/information_schema.py
+++ b/tests/system-test/0-others/information_schema.py
@@ -222,7 +222,7 @@ class TDTestCase:
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
         tdLog.info(len(tdSql.queryResult))
-        tdSql.checkEqual(True, len(tdSql.queryResult) in range(258, 259))
+        tdSql.checkEqual(True, len(tdSql.queryResult) in range(259, 260))
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
         tdSql.checkEqual(54, len(tdSql.queryResult))

From 96e148778705a826cd3922dfc091f6fcb2968417 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 16:57:09 +0800
Subject: [PATCH 025/132] refactor: do some internal refactor.

---
 source/libs/stream/src/streamMeta.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 7f1128b929..519800be28 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -151,10 +151,10 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) {
   int8_t ret = STREAM_STATA_COMPATIBLE;
   TBC*   pCur = NULL;
 
-  if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) {
-    // no task info, no stream
+  if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream
     return ret;
   }
+
   void*   pKey = NULL;
   int32_t kLen = 0;
   void*   pVal = NULL;
@@ -165,20 +165,24 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) {
     if (pVal == NULL || vLen == 0) {
       break;
     }
+
     SDecoder        decoder;
     SCheckpointInfo info;
     tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
     if (tDecodeStreamTaskChkInfo(&decoder, &info) < 0) {
       continue;
     }
+
     if (info.msgVer <= SSTREAM_TASK_INCOMPATIBLE_VER) {
       ret = STREAM_STATA_NO_COMPATIBLE;
     } else if (info.msgVer >= SSTREAM_TASK_NEED_CONVERT_VER) {
       ret = STREAM_STATA_NEED_CONVERT;
     }
+
     tDecoderClear(&decoder);
     break;
   }
+
   tdbFree(pKey);
   tdbFree(pVal);
   tdbTbcClose(pCur);
@@ -193,6 +197,7 @@ int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) {
   if (exist == false) {
     return code;
   }
+
   SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId, pMeta->vgId);
 
   void* pIter = taosHashIterate(pBackend->cfInst, NULL);
@@ -219,6 +224,7 @@ _EXIT:
 
   return code;
 }
+
 int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) {
   int8_t compatible = streamMetaCheckBackendCompatible(pMeta);
   if (compatible == STREAM_STATA_COMPATIBLE) {
@@ -854,6 +860,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
 
   tdbTbcMoveToFirst(pCur);
   while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
+    if (pVal == NULL || vLen == 0) {
+      break;
+    }
+
     SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
     if (pTask == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;

From f146ae4198287a7a305604794b4df8b473910c13 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 22 May 2024 19:46:24 +0800
Subject: [PATCH 026/132] fix(stream):

---
 source/libs/stream/src/streamDispatch.c | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index f3569d8973..7c80d15307 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -1023,7 +1023,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
 
       // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state
       if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) {
-        stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to prepare transfer state", id, msgId);
+        stDebug("s-task:%s dispatch trans-state msgId:%d to downstream successfully, start to prepare transfer state", id, msgId);
         ASSERT(pTask->info.fillHistory == 1);
 
         code = streamTransferStatePrepare(pTask);
@@ -1121,21 +1121,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
           streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId);
           stDebug("s-task:%s close inputQ for upstream:0x%x, msgId:%d", id, pReq->upstreamTaskId, pReq->msgId);
         } else if (pReq->type == STREAM_INPUT__TRANS_STATE) {
-          atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
-          streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId);
-
-          // disable the related stream task here to avoid it to receive the newly arrived data after the transfer-state
-          STaskId* pRelTaskId = &pTask->streamTaskId;
-          SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pRelTaskId->streamId, pRelTaskId->taskId);
-          if (pStreamTask != NULL) {
-            atomic_add_fetch_32(&pStreamTask->upstreamInfo.numOfClosed, 1);
-            streamTaskCloseUpstreamInput(pStreamTask, pReq->upstreamRelTaskId);
-            streamMetaReleaseTask(pMeta, pStreamTask);
-          }
-
-          stDebug("s-task:%s close inputQ for upstream:0x%x since trans-state msgId:%d recv, rel stream-task:0x%" PRIx64
-                      " close inputQ for upstream:0x%x",
-                  id, pReq->upstreamTaskId, pReq->msgId, pTask->streamTaskId.taskId, pReq->upstreamRelTaskId);
+          stDebug("s-task:%s recv trans-state msgId:%d from upstream:0x%x", id, pReq->msgId, pReq->upstreamTaskId);
         }
 
         status = streamTaskAppendInputBlocks(pTask, pReq);

From 61b08259ba2672c186dd760785660ffb4dd014bc Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 May 2024 15:38:16 +0800
Subject: [PATCH 027/132] fix(stream): initialization fill-history tasks before
 start all other stream tasks.

---
 source/libs/stream/src/streamMeta.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 519800be28..190c60289f 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1392,6 +1392,33 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
 
   // broadcast the check downstream tasks msg
   numOfTasks = taosArrayGetSize(pTaskList);
+
+  // prepare the fill-history task before starting all stream tasks, to avoid fill-history tasks are started without
+  // initialization , when the operation of check downstream tasks status is executed far quickly.
+  for(int32_t i = 0; i < numOfTasks; ++i) {
+    SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
+    SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId);
+    if (pTask == NULL) {
+      stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId);
+      streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId);
+      continue;
+    }
+
+    if (pTask->info.fillHistory == 1) {
+      if (pTask->pBackend == NULL) { // TODO: add test cases for this
+        code = expandFn(pTask);
+        if (code != TSDB_CODE_SUCCESS) {
+          stError("s-task:0x%x vgId:%d failed to expand stream backend", pTaskId->taskId, vgId);
+          streamMetaAddFailedTaskSelf(pTask, pTask->execInfo.readyTs);
+        }
+      } else {
+        stDebug("s-task:0x%x vgId:%d fill-history task backend has initializied already", pTaskId->taskId, vgId);
+      }
+    }
+
+    streamMetaReleaseTask(pMeta, pTask);
+  }
+
   for (int32_t i = 0; i < numOfTasks; ++i) {
     SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
 

From 69567799ebae4487bbde103474d323fff9c9253a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 23 May 2024 15:45:59 +0800
Subject: [PATCH 028/132] fix(stream): initialization tasks before start all
 tasks.

---
 source/libs/stream/src/streamMeta.c | 29 +++++++++--------------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 190c60289f..00fe1207dd 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1395,25 +1395,23 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
 
   // prepare the fill-history task before starting all stream tasks, to avoid fill-history tasks are started without
   // initialization , when the operation of check downstream tasks status is executed far quickly.
-  for(int32_t i = 0; i < numOfTasks; ++i) {
+  for (int32_t i = 0; i < numOfTasks; ++i) {
     SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
-    SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId);
+    SStreamTask*   pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId);
     if (pTask == NULL) {
       stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId);
       streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId);
       continue;
     }
 
-    if (pTask->info.fillHistory == 1) {
-      if (pTask->pBackend == NULL) { // TODO: add test cases for this
-        code = expandFn(pTask);
-        if (code != TSDB_CODE_SUCCESS) {
-          stError("s-task:0x%x vgId:%d failed to expand stream backend", pTaskId->taskId, vgId);
-          streamMetaAddFailedTaskSelf(pTask, pTask->execInfo.readyTs);
-        }
-      } else {
-        stDebug("s-task:0x%x vgId:%d fill-history task backend has initializied already", pTaskId->taskId, vgId);
+    if (pTask->pBackend == NULL) {  // TODO: add test cases for this
+      code = expandFn(pTask);
+      if (code != TSDB_CODE_SUCCESS) {
+        stError("s-task:0x%x vgId:%d failed to expand stream backend", pTaskId->taskId, vgId);
+        streamMetaAddFailedTaskSelf(pTask, pTask->execInfo.readyTs);
       }
+    } else {
+      stDebug("s-task:0x%x vgId:%d fill-history task backend has initialized already", pTaskId->taskId, vgId);
     }
 
     streamMetaReleaseTask(pMeta, pTask);
@@ -1430,15 +1428,6 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
     }
 
     STaskExecStatisInfo* pInfo = &pTask->execInfo;
-    if (pTask->pBackend == NULL) {
-      code = expandFn(pTask);
-      if (code != TSDB_CODE_SUCCESS) {
-        stError("s-task:0x%x vgId:%d failed to expand stream backend", pTaskId->taskId, vgId);
-        streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
-        streamMetaReleaseTask(pMeta, pTask);
-        continue;
-      }
-    }
 
     // fill-history task can only be launched by related stream tasks.
     if (pTask->info.fillHistory == 1) {

From 8ccfa728f89d43f1ab07e8935f824a2c745e0650 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 23 May 2024 16:17:15 +0800
Subject: [PATCH 029/132] set pk column

---
 source/libs/executor/src/streamtimewindowoperator.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 2da9ed0353..8b24c1f591 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -855,7 +855,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
   int32_t pkLen = 0;
   SColumnInfoData* pPkColDataInfo = NULL;
   if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
-    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
+    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
   }
 
   if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {
@@ -2142,7 +2142,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
   int32_t pkLen = 0;
   SColumnInfoData* pPkColDataInfo = NULL;
   if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
-    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
+    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
   }
 
   for (int32_t i = 0; i < rows;) {

From 441a150d6615bb89cfa1d285ac0f7bbf2d4eb190 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 23 May 2024 17:35:54 +0800
Subject: [PATCH 030/132] optimize tmq snapshot meta

---
 include/client/taos.h                   |   1 +
 include/common/tcommon.h                |   1 +
 include/common/tmsg.h                   |  14 ++
 include/libs/executor/executor.h        |   2 +-
 include/util/tencode.h                  |   1 +
 source/client/inc/clientInt.h           |  15 +-
 source/client/src/clientMain.c          |  34 +++--
 source/client/src/clientRawBlockWrite.c | 181 ++++++++++++++++++------
 source/client/src/clientTmq.c           |  64 +++++++--
 source/common/src/tmsg.c                |  52 +++++++
 source/dnode/vnode/src/inc/tq.h         |   2 +-
 source/dnode/vnode/src/tq/tqScan.c      |   8 +-
 source/dnode/vnode/src/tq/tqUtil.c      |  50 ++++++-
 source/libs/executor/inc/querytask.h    |   2 +-
 source/libs/executor/src/executor.c     |   4 +-
 source/libs/executor/src/scanoperator.c |  77 +++++++---
 utils/test/c/tmq_taosx_ci.c             |   1 +
 17 files changed, 399 insertions(+), 110 deletions(-)

diff --git a/include/client/taos.h b/include/client/taos.h
index 45dc85f6d9..66b5b92869 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -283,6 +283,7 @@ typedef enum tmq_res_t {
   TMQ_RES_DATA = 1,
   TMQ_RES_TABLE_META = 2,
   TMQ_RES_METADATA = 3,
+  TMQ_RES_BATCH_TABLE_META = 4,
 } tmq_res_t;
 
 typedef struct tmq_topic_assignment {
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index d28477ae40..a5a805ff40 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -144,6 +144,7 @@ enum {
   TMQ_MSG_TYPE__EP_RSP,
   TMQ_MSG_TYPE__POLL_DATA_META_RSP,
   TMQ_MSG_TYPE__WALINFO_RSP,
+  TMQ_MSG_TYPE__POLL_BATCH_META_RSP,
 };
 
 enum {
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 3ed6b40d4d..1657a17ff0 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3978,6 +3978,20 @@ int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp);
 int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp);
 void    tDeleteSTaosxRsp(void* pRsp);
 
+typedef struct SMqBatchMetaRsp {
+  SMqRspHead   head;         // not serialize
+  STqOffsetVal rspOffset;
+  SArray*      batchMetaLen;
+  SArray*      batchMetaReq;
+  void*        pMetaBuff;    // not serialize
+  uint32_t     metaBuffLen;  // not serialize
+} SMqBatchMetaRsp;
+
+int32_t tEncodeMqBatchMetaRsp(SEncoder* pEncoder, const SMqBatchMetaRsp* pRsp);
+int32_t tDecodeMqBatchMetaRsp(SDecoder* pDecoder, SMqBatchMetaRsp* pRsp);
+int32_t tSemiDecodeMqBatchMetaRsp(SDecoder* pDecoder, SMqBatchMetaRsp* pRsp);
+void    tDeleteMqBatchMetaRsp(SMqBatchMetaRsp* pRsp);
+
 typedef struct {
   SMqRspHead head;
   char       cgroup[TSDB_CGROUP_LEN];
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index fe20639c77..9b557500f0 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -201,7 +201,7 @@ void qStreamSetSourceExcluded(qTaskInfo_t tinfo, int8_t sourceExcluded);
 
 void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);
 
-SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
+SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
 
 const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
 
diff --git a/include/util/tencode.h b/include/util/tencode.h
index 596fa2b4d3..b8da040689 100644
--- a/include/util/tencode.h
+++ b/include/util/tencode.h
@@ -77,6 +77,7 @@ typedef struct {
 #define TD_CODER_CURRENT(CODER)                        ((CODER)->data + (CODER)->pos)
 #define TD_CODER_MOVE_POS(CODER, MOVE)                 ((CODER)->pos += (MOVE))
 #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
+#define TD_CODER_REMAIN_CAPACITY(CODER)                ((CODER)->size - (CODER)->pos)
 
 #define tEncodeSize(E, S, SIZE, RET) \
   do {                               \
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 9507472df0..577b7d7ac4 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -44,6 +44,7 @@ enum {
   RES_TYPE__TMQ,
   RES_TYPE__TMQ_META,
   RES_TYPE__TMQ_METADATA,
+  RES_TYPE__TMQ_BATCH_META,
 };
 
 #define SHOW_VARIABLES_RESULT_COLS       3
@@ -51,10 +52,11 @@ enum {
 #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
 #define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
 
-#define TD_RES_QUERY(res)        (*(int8_t*)res == RES_TYPE__QUERY)
-#define TD_RES_TMQ(res)          (*(int8_t*)res == RES_TYPE__TMQ)
-#define TD_RES_TMQ_META(res)     (*(int8_t*)res == RES_TYPE__TMQ_META)
-#define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA)
+#define TD_RES_QUERY(res)          (*(int8_t*)res == RES_TYPE__QUERY)
+#define TD_RES_TMQ(res)            (*(int8_t*)res == RES_TYPE__TMQ)
+#define TD_RES_TMQ_META(res)       (*(int8_t*)res == RES_TYPE__TMQ_META)
+#define TD_RES_TMQ_METADATA(res)   (*(int8_t*)res == RES_TYPE__TMQ_METADATA)
+#define TD_RES_TMQ_BATCH_META(res) (*(int8_t*)res == RES_TYPE__TMQ_BATCH_META)
 
 typedef struct SAppInstInfo SAppInstInfo;
 
@@ -241,6 +243,11 @@ typedef struct {
   STaosxRsp       rsp;
 } SMqTaosxRspObj;
 
+typedef struct {
+  SMqRspObjCommon common;
+  SMqBatchMetaRsp rsp;
+} SMqBatchMetaRspObj;
+
 typedef struct SReqRelInfo {
   uint64_t userRefId;
   uint64_t prevRefId;
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index ba7f65c52b..da5da044a7 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -301,7 +301,7 @@ void taos_close(TAOS *taos) {
 }
 
 int taos_errno(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return terrno;
   }
 
@@ -313,7 +313,7 @@ int taos_errno(TAOS_RES *res) {
 }
 
 const char *taos_errstr(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return (const char *)tstrerror(terrno);
   }
 
@@ -354,6 +354,10 @@ void taos_free_result(TAOS_RES *res) {
     SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res;
     tDeleteMqMetaRsp(&pRspObj->metaRsp);
     taosMemoryFree(pRspObj);
+  } else if (TD_RES_TMQ_BATCH_META(res)) {
+    SMqBatchMetaRspObj *pBtRspObj = (SMqBatchMetaRspObj *)res;
+    tDeleteMqBatchMetaRsp(&pBtRspObj->rsp);
+    taosMemoryFree(pBtRspObj);
   }
 }
 
@@ -371,7 +375,7 @@ void taos_kill_query(TAOS *taos) {
 }
 
 int taos_field_count(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return 0;
   }
 
@@ -382,7 +386,7 @@ int taos_field_count(TAOS_RES *res) {
 int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
 
 TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
-  if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res)) {
+  if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return NULL;
   }
 
@@ -437,7 +441,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
       pResultInfo->current += 1;
       return pResultInfo->row;
     }
-  } else if (TD_RES_TMQ_META(res)) {
+  } else if (TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return NULL;
   } else {
     // assert to avoid un-initialization error
@@ -548,7 +552,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
 }
 
 int *taos_fetch_lengths(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return NULL;
   }
 
@@ -557,7 +561,7 @@ int *taos_fetch_lengths(TAOS_RES *res) {
 }
 
 TAOS_ROW *taos_result_block(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     terrno = TSDB_CODE_INVALID_PARA;
     return NULL;
   }
@@ -625,7 +629,7 @@ const char *taos_get_client_info() { return version; }
 
 // return int32_t
 int taos_affected_rows(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
+  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
     return 0;
   }
 
@@ -636,7 +640,7 @@ int taos_affected_rows(TAOS_RES *res) {
 
 // return int64_t
 int64_t taos_affected_rows64(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
+  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
     return 0;
   }
 
@@ -646,7 +650,7 @@ int64_t taos_affected_rows64(TAOS_RES *res) {
 }
 
 int taos_result_precision(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return TSDB_TIME_PRECISION_MILLI;
   }
 
@@ -686,7 +690,7 @@ int taos_select_db(TAOS *taos, const char *db) {
 }
 
 void taos_stop_query(TAOS_RES *res) {
-  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) {
+  if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
     return;
   }
 
@@ -694,7 +698,7 @@ void taos_stop_query(TAOS_RES *res) {
 }
 
 bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return true;
   }
   SReqResultInfo *pResultInfo = tscGetCurResInfo(res);
@@ -719,7 +723,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
 }
 
 int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return 0;
   }
 
@@ -761,7 +765,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
   *numOfRows = 0;
   *pData = NULL;
 
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return 0;
   }
 
@@ -797,7 +801,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
 }
 
 int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
-  if (res == NULL || TD_RES_TMQ_META(res)) {
+  if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
     return 0;
   }
 
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index c5069d46aa..a37cd99360 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -26,6 +26,8 @@
 #define LOG_ID_TAG   "connId:0x%" PRIx64 ",reqId:0x%" PRIx64
 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId
 
+static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
+
 static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
 
 static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
@@ -2025,38 +2027,81 @@ end:
   return code;
 }
 
+static char* processSimpleMeta(SMqMetaRsp* pMetaRsp) {
+  if (pMetaRsp->resMsgType == TDMT_VND_CREATE_STB) {
+    return processCreateStb(pMetaRsp);
+  } else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_STB) {
+    return processAlterStb(pMetaRsp);
+  } else if (pMetaRsp->resMsgType == TDMT_VND_DROP_STB) {
+    return processDropSTable(pMetaRsp);
+  } else if (pMetaRsp->resMsgType == TDMT_VND_CREATE_TABLE) {
+    return processCreateTable(pMetaRsp);
+  } else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_TABLE) {
+    return processAlterTable(pMetaRsp);
+  } else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) {
+    return processDropTable(pMetaRsp);
+  } else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) {
+    return processDropTable(pMetaRsp);
+  } else if (pMetaRsp->resMsgType == TDMT_VND_DELETE) {
+    return processDeleteTable(pMetaRsp);
+  }
+
+  return NULL;
+}
+static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) {
+  SDecoder        coder;
+  SMqBatchMetaRsp rsp = {0};
+  tDecoderInit(&coder, pMsgRsp->pMetaBuff, pMsgRsp->metaBuffLen);
+  if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) {
+    goto _end;
+  }
+
+  int64_t fullSize = 0;
+  int32_t num = taosArrayGetSize(rsp.batchMetaReq);
+  SArray* strArray = taosArrayInit(num, POINTER_BYTES);
+  for (int32_t i = 0; i < num; i++) {
+    int32_t len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i);
+    void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
+    SDecoder metaCoder = {0};
+    SMqMetaRsp metaRsp = {0};
+    tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), len - sizeof(SMqRspHead));
+    if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) {
+      goto _end;
+    }
+    char* subStr = processSimpleMeta(&metaRsp);
+    tDeleteMqMetaRsp(&metaRsp);
+    fullSize += strlen(subStr);
+    taosArrayPush(strArray, &subStr);
+  }
+
+  char* buf = (char*)taosMemoryCalloc(1, fullSize + num + 1);
+  for (int32_t i = 0; i < num; i++) {
+    char* subStr = taosArrayGetP(strArray, i);
+    strcat(buf, subStr);
+    strcat(buf, "\n");
+  }
+
+_end:
+  return NULL;
+}
+
 char* tmq_get_json_meta(TAOS_RES* res) {
   if (res == NULL) return NULL;
   uDebug("tmq_get_json_meta res:%p", res);
-  if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) {
+  if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res) && !TD_RES_TMQ_BATCH_META(res)) {
     return NULL;
   }
 
   if (TD_RES_TMQ_METADATA(res)) {
     SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res;
     return processAutoCreateTable(&pMetaDataRspObj->rsp);
+  } else if (TD_RES_TMQ_BATCH_META(res)) {
+    SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res;
+    return processBatchMetaToJson(&pBatchMetaRspObj->rsp);
   }
 
   SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
-  if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) {
-    return processCreateStb(&pMetaRspObj->metaRsp);
-  } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) {
-    return processAlterStb(&pMetaRspObj->metaRsp);
-  } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) {
-    return processDropSTable(&pMetaRspObj->metaRsp);
-  } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) {
-    return processCreateTable(&pMetaRspObj->metaRsp);
-  } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) {
-    return processAlterTable(&pMetaRspObj->metaRsp);
-  } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) {
-    return processDropTable(&pMetaRspObj->metaRsp);
-  } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) {
-    return processDropTable(&pMetaRspObj->metaRsp);
-  } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DELETE) {
-    return processDeleteTable(&pMetaRspObj->metaRsp);
-  }
-
-  return NULL;
+  return processSimpleMeta(&pMetaRspObj->metaRsp);
 }
 
 void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
@@ -2147,6 +2192,12 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
     }
     raw->raw_type = RES_TYPE__TMQ_METADATA;
     uDebug("tmq get raw type metadata:%p", raw);
+  } else if (TD_RES_TMQ_BATCH_META(res)) {
+    SMqBatchMetaRspObj* pBtMetaRspObj = (SMqBatchMetaRspObj*)res;
+    raw->raw = pBtMetaRspObj->rsp.pMetaBuff;
+    raw->raw_len = pBtMetaRspObj->rsp.metaBuffLen;
+    raw->raw_type = RES_TYPE__TMQ_BATCH_META;
+    uDebug("tmq get raw batch meta:%p", raw);
   } else {
     uError("tmq get raw error type:%d", *(int8_t*)res);
     terrno = TSDB_CODE_TMQ_INVALID_MSG;
@@ -2163,32 +2214,74 @@ void tmq_free_raw(tmq_raw_data raw) {
   memset(terrMsg, 0, ERR_MSG_LEN);
 }
 
+static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
+  if (type == TDMT_VND_CREATE_STB) {
+    return taosCreateStb(taos, buf, len);
+  } else if (type == TDMT_VND_ALTER_STB) {
+    return taosCreateStb(taos, buf, len);
+  } else if (type == TDMT_VND_DROP_STB) {
+    return taosDropStb(taos, buf, len);
+  } else if (type == TDMT_VND_CREATE_TABLE) {
+    return taosCreateTable(taos, buf, len);
+  } else if (type == TDMT_VND_ALTER_TABLE) {
+    return taosAlterTable(taos, buf, len);
+  } else if (type == TDMT_VND_DROP_TABLE) {
+    return taosDropTable(taos, buf, len);
+  } else if (type == TDMT_VND_DELETE) {
+    return taosDeleteData(taos, buf, len);
+  } else if (type == RES_TYPE__TMQ) {
+    return tmqWriteRawDataImpl(taos, buf, len);
+  } else if (type == RES_TYPE__TMQ_METADATA) {
+    return tmqWriteRawMetaDataImpl(taos, buf, len);
+  } else if (type == RES_TYPE__TMQ_BATCH_META) {
+    return tmqWriteBatchMetaDataImpl(taos, buf, len);
+  }
+  return TSDB_CODE_INVALID_PARA;
+}
+
 int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
   if (!taos) {
-    goto end;
+    return TSDB_CODE_INVALID_PARA;
   }
 
-  if (raw.raw_type == TDMT_VND_CREATE_STB) {
-    return taosCreateStb(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == TDMT_VND_ALTER_STB) {
-    return taosCreateStb(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == TDMT_VND_DROP_STB) {
-    return taosDropStb(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == TDMT_VND_CREATE_TABLE) {
-    return taosCreateTable(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == TDMT_VND_ALTER_TABLE) {
-    return taosAlterTable(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == TDMT_VND_DROP_TABLE) {
-    return taosDropTable(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == TDMT_VND_DELETE) {
-    return taosDeleteData(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == RES_TYPE__TMQ) {
-    return tmqWriteRawDataImpl(taos, raw.raw, raw.raw_len);
-  } else if (raw.raw_type == RES_TYPE__TMQ_METADATA) {
-    return tmqWriteRawMetaDataImpl(taos, raw.raw, raw.raw_len);
-  }
-
-end:
-  terrno = TSDB_CODE_INVALID_PARA;
-  return terrno;
+  return writeRawImpl(taos, raw.raw, raw.raw_len, raw.raw_type);
+}
+
+static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen) {
+  if (taos == NULL || meta == NULL) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  SMqBatchMetaRsp rsp = {0};
+  SDecoder        coder;
+  int32_t         code = TSDB_CODE_SUCCESS;
+
+  // decode and process req
+  tDecoderInit(&coder, meta, metaLen);
+  if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) {
+    code = TSDB_CODE_INVALID_PARA;
+    goto _end;
+  }
+  int32_t num = taosArrayGetSize(rsp.batchMetaReq);
+  for (int32_t i = 0; i < num; i++) {
+    int32_t    len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i);
+    void*      tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
+    SDecoder   metaCoder = {0};
+    SMqMetaRsp metaRsp = {0};
+    tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), len - sizeof(SMqRspHead));
+    if (tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0) {
+      code = TSDB_CODE_INVALID_PARA;
+      goto _end;
+    }
+    code = writeRawImpl(taos, metaRsp.metaRsp, metaRsp.metaRspLen, metaRsp.resMsgType);
+    tDeleteMqMetaRsp(&metaRsp);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _end;
+    }
+  }
+
+_end:
+  tDeleteMqBatchMetaRsp(&rsp);
+  errno = code;
+  return code;
 }
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 3141e21f26..df08a66bd3 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -171,9 +171,10 @@ typedef struct {
   uint64_t        reqId;
   SEpSet*         pEpset;
   union {
-    SMqDataRsp dataRsp;
-    SMqMetaRsp metaRsp;
-    STaosxRsp  taosxRsp;
+    SMqDataRsp      dataRsp;
+    SMqMetaRsp      metaRsp;
+    STaosxRsp       taosxRsp;
+    SMqBatchMetaRsp batchMetaRsp;
   };
 } SMqPollRspWrapper;
 
@@ -635,6 +636,11 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c
     pTopicName = pRspObj->common.topic;
     vgId = pRspObj->common.vgId;
     offsetVal = pRspObj->rsp.common.rspOffset;
+  } else if (TD_RES_TMQ_BATCH_META(pRes)) {
+    SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)pRes;
+    pTopicName = pBtRspObj->common.topic;
+    vgId = pBtRspObj->common.vgId;
+    offsetVal = pBtRspObj->rsp.rspOffset;
   } else {
     code = TSDB_CODE_TMQ_INVALID_MSG;
     goto end;
@@ -934,7 +940,11 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
     SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
     taosMemoryFreeClear(pRsp->pEpset);
     tDeleteSTaosxRsp(&pRsp->taosxRsp);
-  }
+  } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
+    SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
+    taosMemoryFreeClear(pRsp->pEpset);
+    tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp);
+  } 
 }
 
 void tmqClearUnhandleMsg(tmq_t* tmq) {
@@ -1418,6 +1428,17 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     }
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead));
+  } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
+    SDecoder decoder;
+    tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
+    if(tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0){
+      tDecoderClear(&decoder);
+      taosReleaseRef(tmqMgmt.rsetId, refId);
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto FAIL;
+    }
+    tDecoderClear(&decoder);
+    memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead));
   } else {  // invalid rspType
     tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
   }
@@ -1616,6 +1637,19 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
+SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
+  SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj));
+  if(pRspObj == NULL) {
+    return NULL;
+  }
+  pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META;
+  tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
+  tstrncpy(pRspObj->common.db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
+  pRspObj->common.vgId = pWrapper->vgHandle->vgId;
+
+  memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp));
+  return pRspObj;
+}
 
 void changeByteEndian(char* pData){
   char* p = pData;
@@ -1972,7 +2006,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
         tmqFreeRspWrapper(pRspWrapper);
         taosFreeQitem(pRspWrapper);
       }
-    } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
+    } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
       int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
 
@@ -1995,7 +2029,12 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
         updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset,
                      pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true);
         // build rsp
-        SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
+        void* pRsp = NULL;
+        if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
+          pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
+        } else {
+          pRsp = tmqBuildBatchMetaRspFromWrapper(pollRspWrapper);
+        }
         taosFreeQitem(pRspWrapper);
         taosWUnLockLatch(&tmq->lock);
         return pRsp;
@@ -2217,6 +2256,8 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
     return TMQ_RES_TABLE_META;
   } else if (TD_RES_TMQ_METADATA(res)) {
     return TMQ_RES_METADATA;
+  } else if (TD_RES_TMQ_BATCH_META(res)) {
+    return TMQ_RES_BATCH_TABLE_META;
   } else {
     return TMQ_RES_INVALID;
   }
@@ -2226,7 +2267,7 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
   if (res == NULL) {
     return NULL;
   }
-  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
+  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
     return strchr(((SMqRspObjCommon*)res)->topic, '.') + 1;
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
@@ -2241,7 +2282,7 @@ const char* tmq_get_db_name(TAOS_RES* res) {
     return NULL;
   }
 
-  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
+  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
     return strchr(((SMqRspObjCommon*)res)->db, '.') + 1;
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
@@ -2255,7 +2296,7 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
   if (res == NULL) {
     return -1;
   }
-  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
+  if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
     return ((SMqRspObjCommon*)res)->vgId;
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
@@ -2282,6 +2323,11 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) {
     if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) {
       return pRspObj->metaRsp.rspOffset.version;
     }
+  } else if (TD_RES_TMQ_BATCH_META(res)) {
+    SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)res;
+    if (pBtRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) {
+      return pBtRspObj->rsp.rspOffset.version;
+    }
   } else {
     tscError("invalid tmq type:%d", *(int8_t*)res);
   }
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index b7d1417451..3f76d03289 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -10557,3 +10557,55 @@ void tFreeFetchTtlExpiredTbsRsp(void *p) {
   SVFetchTtlExpiredTbsRsp *pRsp = p;
   taosArrayDestroy(pRsp->pExpiredTbs);
 }
+
+int32_t tEncodeMqBatchMetaRsp(SEncoder *pEncoder, const SMqBatchMetaRsp *pRsp) {
+  if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;
+
+  int32_t size = taosArrayGetSize(pRsp->batchMetaReq);
+  if (tEncodeI32(pEncoder, size) < 0) return -1;
+  if (size > 0) {
+    for (int32_t i = 0; i < size; i++) {
+      void   *pMetaReq = taosArrayGetP(pRsp->batchMetaReq, i);
+      int32_t metaLen = *(int32_t *)taosArrayGet(pRsp->batchMetaLen, i);
+      if (tEncodeBinary(pEncoder, pMetaReq, metaLen) < 0) return -1;
+    }
+  }
+  return 0;
+}
+
+int32_t tDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
+  int32_t size = 0;
+  if (tDecodeI32(pDecoder, &size) < 0) return -1;
+  if (size > 0) {
+    pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES);
+    pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t));
+    for (int32_t i = 0; i < size; i++) {
+      void    *pCreate = NULL;
+      uint64_t len = 0;
+      if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
+      int32_t l = (int32_t)len;
+      taosArrayPush(pRsp->batchMetaReq, &pCreate);
+      taosArrayPush(pRsp->batchMetaLen, &l);
+    }
+  }
+  return 0;
+}
+
+int32_t tSemiDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
+  if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1;
+  if (pDecoder->size < pDecoder->pos) {
+    return -1;
+  }
+  pRsp->metaBuffLen = TD_CODER_REMAIN_CAPACITY(pDecoder);
+  pRsp->pMetaBuff = taosMemoryCalloc(1, pRsp->metaBuffLen);
+  memcpy(pRsp->pMetaBuff, TD_CODER_CURRENT(pDecoder), pRsp->metaBuffLen);
+  return 0;
+}
+
+void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp *pRsp) {
+  taosMemoryFreeClear(pRsp->pMetaBuff);
+  taosArrayDestroyP(pRsp->batchMetaReq, taosMemoryFree);
+  taosArrayDestroy(pRsp->batchMetaLen);
+  pRsp->batchMetaReq = NULL;
+  pRsp->batchMetaLen = NULL;
+}
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index e2ecdca59f..08d32b2b81 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -113,7 +113,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
 void    tqDestroyTqHandle(void* data);
 
 // tqRead
-int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
+int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset);
 int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest);
 int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId);
 
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index 08f1689f2f..b9c9595c74 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -163,7 +163,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
   return 0;
 }
 
-int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
+int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) {
   const STqExecHandle* pExec = &pHandle->execHandle;
   qTaskInfo_t          task = pExec->task;
 
@@ -218,10 +218,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
     }
 
     // get meta
-    SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
-    if (tmp->metaRspLen > 0) {
+    SMqBatchMetaRsp* tmp = qStreamExtractMetaMsg(task);
+    if (taosArrayGetSize(tmp->batchMetaReq) > 0) {
       qStreamExtractOffset(task, &tmp->rspOffset);
-      *pMetaRsp = *tmp;
+      *pBatchMetaRsp = *tmp;
 
       tqDebug("tmqsnap task get meta");
       break;
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 28f4a19949..df7539bb69 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -17,6 +17,8 @@
 
 static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
                                  const SMqMetaRsp* pRsp, int32_t vgId);
+static int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
+                                      const SMqBatchMetaRsp* pRsp, int32_t vgId);
 
 int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) {
   tOffsetCopy(&pRsp->reqOffset, &pOffset);
@@ -187,7 +189,6 @@ end : {
   } \
   tDecoderClear(&decoder);
 
-
 static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
                                                   SRpcMsg* pMsg, STqOffsetVal* offset) {
   int        code = 0;
@@ -197,18 +198,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
   tqInitTaosxRsp(&taosxRsp.common, *offset);
 
   if (offset->type != TMQ_OFFSET__LOG) {
-    if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) {
+    SMqBatchMetaRsp btMetaRsp = {0};
+    if (tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset) < 0) {
       code = -1;
       goto end;
     }
 
-    if (metaRsp.metaRspLen > 0) {
-      code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId);
+    if (taosArrayGetSize(btMetaRsp.batchMetaReq) > 0) {
+      code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
       tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64
               ",ts:%" PRId64,
-              pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
-              metaRsp.rspOffset.ts);
-      tDeleteMqMetaRsp(&metaRsp);
+              pRequest->consumerId, pHandle->subKey, vgId, btMetaRsp.rspOffset.type, btMetaRsp.rspOffset.uid,
+              btMetaRsp.rspOffset.ts);
+      tDeleteMqBatchMetaRsp(&btMetaRsp);
       goto end;
     }
 
@@ -352,6 +354,40 @@ static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int
   pMsgHead->walever = ever;
 }
 
+int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp,
+                               int32_t vgId) {
+  int32_t len = 0;
+  int32_t code = 0;
+  tEncodeSize(tEncodeMqBatchMetaRsp, pRsp, len, code);
+  if (code < 0) {
+    return -1;
+  }
+  int32_t tlen = sizeof(SMqRspHead) + len;
+  void*   buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    return -1;
+  }
+
+  int64_t sver = 0, ever = 0;
+  walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever);
+  initMqRspHead(buf, TMQ_MSG_TYPE__POLL_BATCH_META_RSP, pReq->epoch, pReq->consumerId, sver, ever);
+
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, abuf, len);
+  tEncodeMqBatchMetaRsp(&encoder, pRsp);
+  tEncoderClear(&encoder);
+
+  SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
+
+  tmsgSendRsp(&resp);
+  tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, offset type:%d", vgId,
+          pReq->consumerId, pReq->epoch, pRsp->rspOffset.type);
+
+  return 0;
+}
+
 int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp,
                           int32_t vgId) {
   int32_t len = 0;
diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h
index 886ce9705d..18f51df2e9 100644
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@@ -58,7 +58,7 @@ typedef struct STaskStopInfo {
 
 typedef struct {
   STqOffsetVal         currentOffset;  // for tmq
-  SMqMetaRsp           metaRsp;        // for tmq fetching meta
+  SMqBatchMetaRsp      btMetaRsp;      // for tmq fetching meta
   int8_t               sourceExcluded;
   int64_t              snapshotVer;
   SSchemaWrapper*      schema;
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 2f360044c9..69be1c76c7 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1078,9 +1078,9 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) {
   return pTaskInfo->streamInfo.tbName;
 }
 
-SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
+SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  return &pTaskInfo->streamInfo.metaRsp;
+  return &pTaskInfo->streamInfo.btMetaRsp;
 }
 
 void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 110aabf9b1..72ea9afae7 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -44,6 +44,7 @@ int32_t scanDebug = 0;
 #define STREAM_SCAN_OP_NAME               "StreamScanOperator"
 #define STREAM_SCAN_OP_STATE_NAME         "StreamScanFillHistoryState"
 #define STREAM_SCAN_OP_CHECKPOINT_NAME    "StreamScanOperator_Checkpoint"
+#define TMQ_MAX_BATCH_SIZE                4096
 
 typedef struct STableMergeScanExecInfo {
   SFileBlockLoadRecorder blockRecorder;
@@ -2845,8 +2846,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
 
   SStreamRawScanInfo* pInfo = pOperator->info;
   int32_t             code = TSDB_CODE_SUCCESS;
-  pTaskInfo->streamInfo.metaRsp.metaRspLen = 0;  // use metaRspLen !=0 to judge if data is meta
-  pTaskInfo->streamInfo.metaRsp.metaRsp = NULL;
+  pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = NULL;  // use batchMetaReq != NULL to judge if data is meta
+  pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = NULL;
 
   qDebug("tmqsnap doRawScan called");
   if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
@@ -2893,28 +2894,60 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
     return NULL;
   } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) {
     SSnapContext* sContext = pInfo->sContext;
-    void*         data = NULL;
-    int32_t       dataLen = 0;
-    int16_t       type = 0;
-    int64_t       uid = 0;
-    if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) {
-      qError("tmqsnap getTableInfoFromSnapshot error");
-      taosMemoryFreeClear(data);
-      return NULL;
-    }
+    for(int32_t i = 0; i < TMQ_MAX_BATCH_SIZE; i++) {
+      void*         data = NULL;
+      int32_t       dataLen = 0;
+      int16_t       type = 0;
+      int64_t       uid = 0;
+      if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) {
+        qError("tmqsnap getTableInfoFromSnapshot error");
+        taosMemoryFreeClear(data);
+        break;
+      }
 
-    if (!sContext->queryMeta) {  // change to get data next poll request
-      STqOffsetVal offset = {0};
-      SValue val = {0};
-      tqOffsetResetToData(&offset, 0, INT64_MIN, val);
-      qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
-    } else {
-      tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid);
-      pTaskInfo->streamInfo.metaRsp.resMsgType = type;
-      pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen;
-      pTaskInfo->streamInfo.metaRsp.metaRsp = data;
-    }
+      if (!sContext->queryMeta) {  // change to get data next poll request
+        STqOffsetVal offset = {0};
+        SValue val = {0};
+        tqOffsetResetToData(&offset, 0, INT64_MIN, val);
+        qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
+        break;
+      } else {
+        tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid);
+        SMqMetaRsp tmpMetaRsp = {0};
+        tmpMetaRsp.resMsgType = type;
+        tmpMetaRsp.metaRspLen = dataLen;
+        tmpMetaRsp.metaRsp = data;
+        if (!pTaskInfo->streamInfo.btMetaRsp.batchMetaReq) {
+          pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES);
+          pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t));
+        }
+        int32_t  code = TSDB_CODE_SUCCESS;
+        uint32_t len = 0;
+        tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code);
+        if (TSDB_CODE_SUCCESS != code) {
+          qError("tmqsnap tEncodeMqMetaRsp error");
+          taosMemoryFreeClear(data);
+          break;
+        }
 
+        int32_t tLen = sizeof(SMqRspHead) + len;
+        void*   tBuf = taosMemoryCalloc(1, tLen);
+        void*   metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
+        SEncoder encoder = {0};
+        tEncoderInit(&encoder, metaBuff, len);
+        code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
+        if (code < 0) {
+          qError("tmqsnap tEncodeMqMetaRsp error");
+          tEncoderClear(&encoder);
+          taosMemoryFreeClear(tBuf);
+          taosMemoryFreeClear(data);
+          break;
+        }
+        taosMemoryFreeClear(data);
+        taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaReq, &tBuf);
+        taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaLen, &tLen);
+      }
+    }
     return NULL;
   }
   return NULL;
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index a391093609..5642b801f4 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -1138,6 +1138,7 @@ void testConsumeExcluded(int topic_type) {
     taos_close(pConn);
     return;
   }
+  taos_close(pConn);
   taos_free_result(pRes);
 }
 

From f72907204406c99b142d8af621a86441225687ea Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 23 May 2024 17:57:07 +0800
Subject: [PATCH 031/132] enh: reduce compact block write time

---
 source/dnode/vnode/src/inc/tsdb.h           |   9 +-
 source/dnode/vnode/src/inc/vnd.h            |  28 +-
 source/dnode/vnode/src/inc/vnodeInt.h       |  35 +-
 source/dnode/vnode/src/sma/smaRollup.c      |   4 +-
 source/dnode/vnode/src/tsdb/tsdbCommit2.c   | 377 +++++++++++++++-----
 source/dnode/vnode/src/tsdb/tsdbFS2.c       | 127 ++++---
 source/dnode/vnode/src/tsdb/tsdbFS2.h       |  10 +-
 source/dnode/vnode/src/tsdb/tsdbFSet2.c     |  43 ++-
 source/dnode/vnode/src/tsdb/tsdbFSet2.h     |  14 +-
 source/dnode/vnode/src/tsdb/tsdbMerge.c     |   1 +
 source/dnode/vnode/src/tsdb/tsdbRetention.c | 303 ++++++----------
 source/dnode/vnode/src/vnd/vnodeAsync.c     | 169 ++++++---
 source/dnode/vnode/src/vnd/vnodeCommit.c    |  32 +-
 source/dnode/vnode/src/vnd/vnodeHash.h      |  17 -
 source/dnode/vnode/src/vnd/vnodeModule.c    |  15 +-
 source/dnode/vnode/src/vnd/vnodeOpen.c      |   6 +-
 source/dnode/vnode/src/vnd/vnodeRetention.c |  21 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c  |   4 +-
 source/dnode/vnode/src/vnd/vnodeSvr.c       |  28 +-
 source/libs/tfs/src/tfs.c                   |   5 +-
 20 files changed, 722 insertions(+), 526 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index c85316f810..84e1996f2c 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -355,6 +355,8 @@ typedef struct {
   int    flush_count;
 } SCacheFlushState;
 
+typedef struct SCompMonitor SCompMonitor;
+
 struct STsdb {
   char *               path;
   SVnode *             pVnode;
@@ -375,8 +377,11 @@ struct STsdb {
   TdThreadMutex        pgMutex;
   struct STFileSystem *pFS;  // new
   SRocksCache          rCache;
-  // compact monitor
-  struct SCompMonitor *pCompMonitor;
+  SCompMonitor         *pCompMonitor;
+  struct {
+    SVHashTable *ht;
+    SArray      *arr;
+  } *commitInfo;
 };
 
 struct TSDBKEY {
diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h
index 304716744c..7bdc8e1c33 100644
--- a/source/dnode/vnode/src/inc/vnd.h
+++ b/source/dnode/vnode/src/inc/vnd.h
@@ -49,31 +49,21 @@ int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson);
 int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj);
 
 // vnodeAsync.c
-typedef struct SVAsync SVAsync;
-
 typedef enum {
   EVA_PRIORITY_HIGH = 0,
   EVA_PRIORITY_NORMAL,
   EVA_PRIORITY_LOW,
 } EVAPriority;
 
-#define VNODE_ASYNC_VALID_CHANNEL_ID(channelId) ((channelId) > 0)
-#define VNODE_ASYNC_VALID_TASK_ID(taskId)       ((taskId) > 0)
-
-int32_t vnodeAsyncInit(SVAsync** async, char* label);
-int32_t vnodeAsyncDestroy(SVAsync** async);
-int32_t vnodeAChannelInit(SVAsync* async, int64_t* channelId);
-int32_t vnodeAChannelDestroy(SVAsync* async, int64_t channelId, bool waitRunning);
-int32_t vnodeAsync(SVAsync* async, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), void* arg,
-                   int64_t* taskId);
-int32_t vnodeAsyncC(SVAsync* async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void*),
-                    void (*complete)(void*), void* arg, int64_t* taskId);
-int32_t vnodeAWait(SVAsync* async, int64_t taskId);
-int32_t vnodeACancel(SVAsync* async, int64_t taskId);
-int32_t vnodeAsyncSetWorkers(SVAsync* async, int32_t numWorkers);
-
-// vnodeModule.c
-extern SVAsync* vnodeAsyncHandle[2];
+int32_t vnodeAsyncOpen(int32_t numOfThreads);
+int32_t vnodeAsyncClose();
+int32_t vnodeAChannelInit(int64_t async, SVAChannelID* channelID);
+int32_t vnodeAChannelDestroy(SVAChannelID* channelID, bool waitRunning);
+int32_t vnodeAsync(SVAChannelID* channelID, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*),
+                   void* arg, SVATaskID* taskID);
+int32_t vnodeAWait(SVATaskID* taskID);
+int32_t vnodeACancel(SVATaskID* taskID);
+int32_t vnodeAsyncSetWorkers(int64_t async, int32_t numWorkers);
 
 // vnodeBufPool.c
 typedef struct SVBufPoolNode SVBufPoolNode;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 419ebd1a6c..06b58213f7 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -237,9 +237,6 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t
 int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey);
 int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type);
 int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
-int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
-int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync);
-int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync);
 int     tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
 int     tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
 int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
@@ -470,6 +467,16 @@ typedef struct SVMonitorObj {
   taos_counter_t* insertCounter;
 } SVMonitorObj;
 
+typedef struct {
+  int64_t async;
+  int64_t id;
+} SVAChannelID;
+
+typedef struct {
+  int64_t async;
+  int64_t id;
+} SVATaskID;
+
 struct SVnode {
   char*     path;
   SVnodeCfg config;
@@ -491,8 +498,8 @@ struct SVnode {
   SVBufPool*    onRecycle;
 
   // commit variables
-  int64_t commitChannel;
-  int64_t commitTask;
+  SVAChannelID commitChannel;
+  SVATaskID    commitTask;
 
   SMeta*        pMeta;
   SSma*         pSma;
@@ -598,6 +605,24 @@ struct SCompactInfo {
 
 void initStorageAPI(SStorageAPI* pAPI);
 
+// a simple hash table impl
+typedef struct SVHashTable SVHashTable;
+
+struct SVHashTable {
+  uint32_t (*hash)(const void*);
+  int32_t (*compare)(const void*, const void*);
+  int32_t              numEntries;
+  uint32_t             numBuckets;
+  struct SVHashEntry** buckets;
+};
+
+#define vHashNumEntries(ht) ((ht)->numEntries)
+int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*));
+int32_t vHashDestroy(SVHashTable** ht);
+int32_t vHashPut(SVHashTable* ht, void* obj);
+int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj);
+int32_t vHashDrop(SVHashTable* ht, const void* obj);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 5441d0c4c1..7d6eda9cf1 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -622,8 +622,8 @@ int32_t smaRetention(SSma *pSma, int64_t now) {
 
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pSma->pRSmaTsdb[i]) {
-      code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
-      if (code) goto _end;
+      // code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
+      // if (code) goto _end;
     }
   }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index 96c0f2a4ba..ff683a12ba 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -17,13 +17,13 @@
 
 // extern dependencies
 typedef struct {
-  STsdb         *tsdb;
-  TFileSetArray *fsetArr;
-  TFileOpArray   fopArray[1];
-
-  // SSkmInfo skmTb[1];
-  // SSkmInfo skmRow[1];
+  int32_t    fid;
+  bool       hasDataToCommit;
+  STFileSet *fset;
+} SFileSetCommitInfo;
 
+typedef struct {
+  STsdb  *tsdb;
   int32_t minutes;
   int8_t  precision;
   int32_t minRow;
@@ -32,34 +32,33 @@ typedef struct {
   int32_t sttTrigger;
   int32_t szPage;
   int64_t compactVersion;
+  int64_t cid;
+  int64_t now;
 
   struct {
-    int64_t    cid;
-    int64_t    now;
-    TSKEY      nextKey;
-    int32_t    fid;
-    int32_t    expLevel;
-    SDiskID    did;
-    TSKEY      minKey;
-    TSKEY      maxKey;
-    STFileSet *fset;
-    TABLEID    tbid[1];
-    bool       hasTSData;
-    bool       skipTsRow;
-    SHashObj  *pColCmprObj;
+    SFileSetCommitInfo *info;
+
+    int32_t   expLevel;
+    SDiskID   did;
+    TSKEY     minKey;
+    TSKEY     maxKey;
+    TABLEID   tbid[1];
+    bool      hasTSData;
+    bool      skipTsRow;
+    SHashObj *pColCmprObj;
   } ctx[1];
 
   // reader
   TSttFileReaderArray sttReaderArray[1];
-
   // iter
   TTsdbIterArray dataIterArray[1];
   SIterMerger   *dataIterMerger;
   TTsdbIterArray tombIterArray[1];
   SIterMerger   *tombIterMerger;
-
   // writer
   SFSetWriter *writer;
+
+  TFileOpArray fopArray[1];
 } SCommitter2;
 
 static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
@@ -74,8 +73,8 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
       .maxRow = committer->maxRow,
       .szPage = committer->szPage,
       .cmprAlg = committer->cmprAlg,
-      .fid = committer->ctx->fid,
-      .cid = committer->ctx->cid,
+      .fid = committer->ctx->info->fid,
+      .cid = committer->cid,
       .did = committer->ctx->did,
       .level = 0,
   };
@@ -83,11 +82,11 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
   if (committer->sttTrigger == 1) {
     config.toSttOnly = false;
 
-    if (committer->ctx->fset) {
+    if (committer->ctx->info->fset) {
       for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) {
-        if (committer->ctx->fset->farr[ftype] != NULL) {
+        if (committer->ctx->info->fset->farr[ftype] != NULL) {
           config.files[ftype].exist = true;
-          config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0];
+          config.files[ftype].file = committer->ctx->info->fset->farr[ftype]->f[0];
         }
       }
     }
@@ -117,7 +116,7 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
 
   committer->ctx->tbid->suid = 0;
   committer->ctx->tbid->uid = 0;
-  
+
   for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) {
     if (row->uid != committer->ctx->tbid->uid) {
       committer->ctx->tbid->suid = row->suid;
@@ -132,7 +131,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
 
     int64_t ts = TSDBROW_TS(&row->row);
     if (ts > committer->ctx->maxKey) {
-      committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
       code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
       TSDB_CHECK_CODE(code, lino, _exit);
       continue;
@@ -152,7 +150,8 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, numOfRow);
+    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid,
+              numOfRow);
   }
   return code;
 }
@@ -168,7 +167,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
   }
 
   // do not need to write tomb data if there is no ts data
-  bool skip = (committer->ctx->fset == NULL && !committer->ctx->hasTSData);
+  bool skip = (committer->ctx->info->fset == NULL && !committer->ctx->hasTSData);
 
   committer->ctx->tbid->suid = 0;
   committer->ctx->tbid->uid = 0;
@@ -187,12 +186,8 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
     if (record->ekey < committer->ctx->minKey) {
       // do nothing
     } else if (record->skey > committer->ctx->maxKey) {
-      committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey);
+      // committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey);
     } else {
-      if (record->ekey > committer->ctx->maxKey) {
-        committer->ctx->nextKey = TMIN(committer->ctx->nextKey, committer->ctx->maxKey + 1);
-      }
-
       record->skey = TMAX(record->skey, committer->ctx->minKey);
       record->ekey = TMIN(record->ekey, committer->ctx->maxKey);
 
@@ -211,8 +206,8 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->fid,
-              numRecord);
+    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode),
+              committer->ctx->info->fid, numRecord);
   }
   return code;
 }
@@ -223,15 +218,15 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
 
   ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0);
 
-  if (committer->ctx->fset == NULL                        //
-      || committer->sttTrigger > 1                        //
-      || TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 0  //
+  if (committer->ctx->info->fset == NULL                        //
+      || committer->sttTrigger > 1                              //
+      || TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0  //
   ) {
     return 0;
   }
 
   SSttLvl *lvl;
-  TARRAY2_FOREACH(committer->ctx->fset->lvlArr, lvl) {
+  TARRAY2_FOREACH(committer->ctx->info->fset->lvlArr, lvl) {
     STFileObj *fobj = NULL;
     TARRAY2_FOREACH(lvl->fobjArr, fobj) {
       SSttFileReader *sttReader;
@@ -359,22 +354,15 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
   int32_t lino = 0;
   STsdb  *tsdb = committer->tsdb;
 
-  int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
-
   // check if can commit
-  tsdbFSCheckCommit(tsdb, fid);
+  tsdbFSCheckCommit(tsdb, committer->ctx->info->fid);
 
-  committer->ctx->fid = fid;
-  committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now);
-  tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
+  committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now);
+  tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
                   &committer->ctx->maxKey);
   code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did);
   TSDB_CHECK_CODE(code, lino, _exit);
   tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did);
-  STFileSet fset = {.fid = committer->ctx->fid};
-  committer->ctx->fset = &fset;
-  STFileSet **fsetPtr = TARRAY2_SEARCH(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ);
-  committer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr;
   committer->ctx->tbid->suid = 0;
   committer->ctx->tbid->uid = 0;
 
@@ -391,15 +379,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
   code = tsdbCommitOpenWriter(committer);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  // reset nextKey
-  committer->ctx->nextKey = TSKEY_MAX;
-
 _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
   } else {
     tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
-              __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel);
+              __func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey,
+              committer->ctx->expLevel);
   }
   return code;
 }
@@ -421,7 +407,7 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid);
+    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
   }
   return code;
 }
@@ -449,7 +435,201 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid);
+    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
+  }
+  return code;
+}
+
+static int32_t tFileSetCommitInfoCompare(const void *arg1, const void *arg2) {
+  SFileSetCommitInfo *info1 = (SFileSetCommitInfo *)arg1;
+  SFileSetCommitInfo *info2 = (SFileSetCommitInfo *)arg2;
+
+  if (info1->fid < info2->fid) {
+    return -1;
+  } else if (info1->fid > info2->fid) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static int32_t tFileSetCommitInfoPCompare(const void *arg1, const void *arg2) {
+  return tFileSetCommitInfoCompare(*(SFileSetCommitInfo **)arg1, *(SFileSetCommitInfo **)arg2);
+}
+
+static uint32_t tFileSetCommitInfoHash(const void *arg) {
+  SFileSetCommitInfo *info = (SFileSetCommitInfo *)arg;
+  return MurmurHash3_32((const char *)&info->fid, sizeof(info->fid));
+}
+
+static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  if (pTsdb->commitInfo) {
+    for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
+      SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
+      vHashDrop(pTsdb->commitInfo->ht, info);
+      tsdbTFileSetClear(&info->fset);
+      taosMemoryFree(info);
+    }
+
+    vHashDestroy(&pTsdb->commitInfo->ht);
+    taosArrayDestroy(pTsdb->commitInfo->arr);
+    pTsdb->commitInfo->arr = NULL;
+    taosMemoryFreeClear(pTsdb->commitInfo);
+  }
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+  }
+  return code;
+}
+
+static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo));
+  if (pTsdb->commitInfo == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+  }
+
+  code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *));
+  if (pTsdb->commitInfo->arr == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    tsdbCommitInfoDestroy(pTsdb);
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+  }
+  return code;
+}
+
+static int32_t tsdbCommitInfoAdd(STsdb *tsdb, const SFileSetCommitInfo *info) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SFileSetCommitInfo *tinfo;
+
+  vHashGet(tsdb->commitInfo->ht, info, (void **)&tinfo);
+  if (tinfo) {
+    if (info->hasDataToCommit && !tinfo->hasDataToCommit) {
+      tinfo->hasDataToCommit = true;
+    }
+  } else {
+    if ((tinfo = taosMemoryCalloc(1, sizeof(*tinfo))) == NULL) {
+      TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+    }
+    tinfo->fid = info->fid;
+    tinfo->hasDataToCommit = info->hasDataToCommit;
+    if (info->fset) {
+      code = tsdbTFileSetInitCopy(tsdb, info->fset, &tinfo->fset);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+
+    code = vHashPut(tsdb->commitInfo->ht, tinfo);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) {
+      TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+    }
+    taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare);
+  }
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
+  }
+  return code;
+}
+
+static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STFileSet  *fset = NULL;
+  SRBTreeIter iter;
+
+  code = tsdbCommitInfoInit(tsdb);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  taosThreadMutexLock(&tsdb->mutex);
+  TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
+    SFileSetCommitInfo info = {
+        .fid = fset->fid,
+        .hasDataToCommit = false,
+        .fset = fset,
+    };
+    if ((code = tsdbCommitInfoAdd(tsdb, &info))) {
+      taosThreadMutexUnlock(&tsdb->mutex);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+  taosThreadMutexUnlock(&tsdb->mutex);
+
+  iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
+  for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
+    STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
+
+    // scan time-series data
+    STsdbRowKey from = {
+        .key.ts = INT64_MIN,
+        .key.numOfPKs = 0,
+        .version = INT64_MIN,
+    };
+    for (;;) {
+      int64_t     minKey, maxKey;
+      STbDataIter tbDataIter = {0};
+      TSDBROW    *row;
+      int32_t     fid;
+
+      tsdbTbDataIterOpen(pTbData, &from, 0, &tbDataIter);
+      if ((row = tsdbTbDataIterGet(&tbDataIter)) == NULL) {
+        break;
+      }
+
+      fid = tsdbKeyFid(TSDBROW_TS(row), tsdb->keepCfg.days, tsdb->keepCfg.precision);
+      tsdbFidKeyRange(fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
+
+      SFileSetCommitInfo info = {
+          .fid = fid,
+          .hasDataToCommit = true,
+          .fset = NULL,
+      };
+      code = tsdbCommitInfoAdd(tsdb, &info);
+      TSDB_CHECK_CODE(code, lino, _exit);
+
+      from.key.ts = maxKey + 1;
+    }
+
+    // scan tomb data
+    for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) {
+      for (int32_t i = taosArrayGetSize(tsdb->commitInfo->arr) - 1; i >= 0; i--) {
+        int64_t             minKey, maxKey;
+        SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+
+        tsdbFidKeyRange(info->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
+
+        if (pDelData->sKey > maxKey || pDelData->eKey < minKey) {
+          continue;
+        } else if (!info->hasDataToCommit) {
+          info->hasDataToCommit = true;
+        }
+      }
+    }
+  }
+
+_exit:
+  if (code) {
+    tsdbCommitInfoDestroy(tsdb);
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
   }
   return code;
 }
@@ -458,11 +638,7 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
   int32_t code = 0;
   int32_t lino = 0;
 
-  memset(committer, 0, sizeof(committer[0]));
-
   committer->tsdb = tsdb;
-  code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr);
-  TSDB_CHECK_CODE(code, lino, _exit);
   committer->minutes = tsdb->keepCfg.days;
   committer->precision = tsdb->keepCfg.precision;
   committer->minRow = info->info.config.tsdbCfg.minRows;
@@ -471,21 +647,21 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
   committer->sttTrigger = info->info.config.sttTrigger;
   committer->szPage = info->info.config.tsdbPageSize;
   committer->compactVersion = INT64_MAX;
-  committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS);
-  committer->ctx->now = taosGetTimestampSec();
+  committer->cid = tsdbFSAllocEid(tsdb->pFS);
+  committer->now = taosGetTimestampSec();
 
-  committer->ctx->nextKey = tsdb->imem->minKey;
-  if (tsdb->imem->nDel > 0) {
-    SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)};
+  code = tsdbCommitInfoBuild(tsdb);
+  TSDB_CHECK_CODE(code, lino, _exit);
 
-    for (SRBTreeNode *node = tRBTreeIterNext(iter); node; node = tRBTreeIterNext(iter)) {
-      STbData *tbData = TCONTAINER_OF(node, STbData, rbtn);
-
-      for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) {
-        committer->ctx->nextKey = TMIN(committer->ctx->nextKey, delData->sKey);
-      }
+  STFileSet *fset;
+  taosThreadMutexLock(&tsdb->mutex);
+  for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
+    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+    if (info->hasDataToCommit && info->fset) {
+      tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
     }
   }
+  taosThreadMutexUnlock(&tsdb->mutex);
 
 _exit:
   if (code) {
@@ -516,14 +692,13 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
   TARRAY2_DESTROY(committer->sttReaderArray, NULL);
   TARRAY2_DESTROY(committer->fopArray, NULL);
   TARRAY2_DESTROY(committer->sttReaderArray, NULL);
-  tsdbFSDestroyCopySnapshot(&committer->fsetArr);
 
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino,
-              tstrerror(code), committer->ctx->cid);
+              tstrerror(code), committer->cid);
   } else {
-    tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->cid);
+    tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid);
   }
   return code;
 }
@@ -553,17 +728,20 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
     taosThreadMutexUnlock(&tsdb->mutex);
     tsdbUnrefMemTable(imem, NULL, true);
   } else {
-    SCommitter2 committer[1];
+    SCommitter2 committer = {0};
 
-    code = tsdbOpenCommitter(tsdb, info, committer);
+    code = tsdbOpenCommitter(tsdb, info, &committer);
     TSDB_CHECK_CODE(code, lino, _exit);
 
-    while (committer->ctx->nextKey != TSKEY_MAX) {
-      code = tsdbCommitFileSet(committer);
-      TSDB_CHECK_CODE(code, lino, _exit);
+    for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
+      committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+      if (committer.ctx->info->hasDataToCommit) {
+        code = tsdbCommitFileSet(&committer);
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
     }
 
-    code = tsdbCloseCommitter(committer, code);
+    code = tsdbCloseCommitter(&committer, code);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -580,22 +758,29 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  if (tsdb->imem == NULL) goto _exit;
+  if (tsdb->imem) {
+    SMemTable *pMemTable = tsdb->imem;
+    taosThreadMutexLock(&tsdb->mutex);
+    if ((code = tsdbFSEditCommit(tsdb->pFS))) {
+      taosThreadMutexUnlock(&tsdb->mutex);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    tsdb->imem = NULL;
+
+    for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
+      SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+      if (info->hasDataToCommit && info->fset) {
+        tsdbFinishTaskOnFileSet(tsdb, info->fid);
+      }
+    }
 
-  SMemTable *pMemTable = tsdb->imem;
-  taosThreadMutexLock(&tsdb->mutex);
-  code = tsdbFSEditCommit(tsdb->pFS);
-  if (code) {
     taosThreadMutexUnlock(&tsdb->mutex);
-    TSDB_CHECK_CODE(code, lino, _exit);
+    tsdbUnrefMemTable(pMemTable, NULL, true);
   }
-  tsdb->imem = NULL;
-  taosThreadMutexUnlock(&tsdb->mutex);
-  tsdbUnrefMemTable(pMemTable, NULL, true);
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
   } else {
     tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
   }
@@ -611,6 +796,16 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) {
   code = tsdbFSEditAbort(pTsdb->pFS);
   TSDB_CHECK_CODE(code, lino, _exit);
 
+  taosThreadMutexLock(&pTsdb->mutex);
+  for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
+    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
+    if (info->hasDataToCommit && info->fset) {
+      tsdbFinishTaskOnFileSet(pTsdb, info->fid);
+    }
+  }
+  taosThreadMutexUnlock(&pTsdb->mutex);
+  tsdbCommitInfoDestroy(pTsdb);
+
 _exit:
   if (code) {
     tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c
index 3ca26621a1..3087dc44d9 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c
@@ -22,9 +22,6 @@
 
 extern void remove_file(const char *fname);
 
-#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
-#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
-
 typedef struct STFileHashEntry {
   struct STFileHashEntry *next;
   char                    fname[TSDB_FILENAME_LEN];
@@ -290,10 +287,8 @@ static int32_t commit_edit(STFileSystem *fs) {
   current_fname(fs->tsdb, current, TSDB_FCURRENT);
   if (fs->etype == TSDB_FEDIT_COMMIT) {
     current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
-  } else if (fs->etype == TSDB_FEDIT_MERGE) {
-    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
   } else {
-    ASSERT(0);
+    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
   }
 
   int32_t code;
@@ -324,11 +319,8 @@ static int32_t abort_edit(STFileSystem *fs) {
 
   if (fs->etype == TSDB_FEDIT_COMMIT) {
     current_fname(fs->tsdb, fname, TSDB_FCURRENT_C);
-  } else if (fs->etype == TSDB_FEDIT_MERGE) {
-    current_fname(fs->tsdb, fname, TSDB_FCURRENT_M);
   } else {
-    tsdbError("vgId:%d %s failed since invalid etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
-    ASSERT(0);
+    current_fname(fs->tsdb, fname, TSDB_FCURRENT_M);
   }
 
   int32_t code;
@@ -767,9 +759,12 @@ extern int32_t tsdbStopAllCompTask(STsdb *tsdb);
 
 int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
   STFileSystem *fs = pTsdb->pFS;
-  TARRAY2(int64_t) channelArr = {0};
+  SArray       *channelArray = taosArrayInit(0, sizeof(SVAChannelID));
+  if (channelArray == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
 
-  taosThreadMutexLock(&fs->tsdb->mutex);
+  taosThreadMutexLock(&pTsdb->mutex);
 
   // disable
   pTsdb->bgTaskDisabled = true;
@@ -777,20 +772,23 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
   // collect channel
   STFileSet *fset;
   TARRAY2_FOREACH(fs->fSetArr, fset) {
-    if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) {
-      TARRAY2_APPEND(&channelArr, fset->bgTaskChannel);
-      fset->bgTaskChannel = 0;
+    if (fset->channelOpened) {
+      taosArrayPush(channelArray, &fset->channel);
+      fset->channel = (SVAChannelID){0};
+      fset->mergeScheduled = false;
+      tsdbFSSetBlockCommit(fset, false);
+      fset->channelOpened = false;
     }
-    fset->mergeScheduled = false;
-    tsdbFSSetBlockCommit(fset, false);
   }
 
-  taosThreadMutexUnlock(&fs->tsdb->mutex);
+  taosThreadMutexUnlock(&pTsdb->mutex);
 
   // destroy all channels
-  int64_t channel;
-  TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); }
-  TARRAY2_DESTROY(&channelArr, NULL);
+  for (int32_t i = 0; i < taosArrayGetSize(channelArray); i++) {
+    SVAChannelID *channel = taosArrayGet(channelArray, i);
+    vnodeAChannelDestroy(channel, true);
+  }
+  taosArrayDestroy(channelArray);
 
 #ifdef TD_ENTERPRISE
   tsdbStopAllCompTask(pTsdb);
@@ -832,15 +830,10 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e
   int32_t lino;
   char    current_t[TSDB_FILENAME_LEN];
 
-  switch (etype) {
-    case TSDB_FEDIT_COMMIT:
-      current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
-      break;
-    case TSDB_FEDIT_MERGE:
-      current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
-      break;
-    default:
-      ASSERT(0);
+  if (etype == TSDB_FEDIT_COMMIT) {
+    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
+  } else {
+    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
   }
 
   tsem_wait(&fs->canEdit);
@@ -932,8 +925,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
         arg->tsdb = fs->tsdb;
         arg->fid = fset->fid;
 
-        code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg,
-                           NULL);
+        code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, NULL);
         TSDB_CHECK_CODE(code, lino, _exit);
         fset->mergeScheduled = true;
       }
@@ -946,33 +938,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
     }
   }
 
-  // clear empty level and fset
-  int32_t i = 0;
-  while (i < TARRAY2_SIZE(fs->fSetArr)) {
-    STFileSet *fset = TARRAY2_GET(fs->fSetArr, i);
-
-    int32_t j = 0;
-    while (j < TARRAY2_SIZE(fset->lvlArr)) {
-      SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j);
-
-      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
-        TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear);
-      } else {
-        j++;
-      }
-    }
-
-    if (tsdbTFileSetIsEmpty(fset)) {
-      if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) {
-        vnodeAChannelDestroy(vnodeAsyncHandle[1], fset->bgTaskChannel, false);
-        fset->bgTaskChannel = 0;
-      }
-      TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear);
-    } else {
-      i++;
-    }
-  }
-
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code));
@@ -1211,3 +1176,47 @@ _out:
 }
 
 int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); }
+
+int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) {
+  int16_t sttTrigger = tsdb->pVnode->config.sttTrigger;
+
+  tsdbFSGetFSet(tsdb->pFS, fid, fset);
+  if (sttTrigger == 1 && fset) {
+    for (;;) {
+      if ((*fset)->taskRunning) {
+        (*fset)->numWaitTask++;
+
+        taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex);
+
+        tsdbFSGetFSet(tsdb->pFS, fid, fset);
+        ASSERT(fset != NULL);
+
+        (*fset)->numWaitTask--;
+        ASSERT((*fset)->numWaitTask >= 0);
+      } else {
+        (*fset)->taskRunning = true;
+        break;
+      }
+    }
+    tsdbInfo("vgId:%d begin task on file set:%d", TD_VID(tsdb->pVnode), fid);
+  }
+
+  return 0;
+}
+
+int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) {
+  int16_t sttTrigger = tsdb->pVnode->config.sttTrigger;
+  if (sttTrigger == 1) {
+    STFileSet *fset = NULL;
+    tsdbFSGetFSet(tsdb->pFS, fid, &fset);
+    if (fset != NULL && fset->taskRunning) {
+      fset->taskRunning = false;
+      if (fset->numWaitTask > 0) {
+        taosThreadCondSignal(&fset->beginTask);
+      }
+      tsdbInfo("vgId:%d finish task on file set:%d", TD_VID(tsdb->pVnode), fid);
+    }
+  }
+
+  return 0;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h
index 7099ce8b26..6938efde3c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.h
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h
@@ -24,7 +24,9 @@ extern "C" {
 
 typedef enum {
   TSDB_FEDIT_COMMIT = 1,  //
-  TSDB_FEDIT_MERGE
+  TSDB_FEDIT_MERGE,
+  TSDB_FEDIT_COMPACT,
+  TSDB_FEDIT_RETENTION,
 } EFEditT;
 
 typedef enum {
@@ -59,6 +61,8 @@ int32_t tsdbFSEditAbort(STFileSystem *fs);
 // other
 int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset);
 int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid);
+int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset);
+int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid);
 // utils
 int32_t save_fs(const TFileSetArray *arr, const char *fname);
 int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);
@@ -72,10 +76,6 @@ struct STFileSystem {
   EFEditT       etype;
   TFileSetArray fSetArr[1];
   TFileSetArray fSetArrTmp[1];
-
-  // background task queue
-  bool    stop;
-  int64_t taskid;
 };
 
 #ifdef __cplusplus
diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c
index 0820f53117..598ca0aec3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c
@@ -456,13 +456,14 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
   TARRAY2_INIT(fset[0]->lvlArr);
 
   // background task queue
-  fset[0]->bgTaskChannel = 0;
-  fset[0]->mergeScheduled = false;
+  taosThreadCondInit(&(*fset)->beginTask, NULL);
+  (*fset)->taskRunning = false;
+  (*fset)->numWaitTask = 0;
 
   // block commit variables
   taosThreadCondInit(&fset[0]->canCommit, NULL);
-  fset[0]->numWaitCommit = 0;
-  fset[0]->blockCommit = false;
+  (*fset)->numWaitCommit = 0;
+  (*fset)->blockCommit = false;
 
   return 0;
 }
@@ -598,21 +599,19 @@ int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) {
   return 0;
 }
 
-int32_t tsdbTFileSetClear(STFileSet **fset) {
-  if (!fset[0]) return 0;
+void tsdbTFileSetClear(STFileSet **fset) {
+  if (fset && *fset) {
+    for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
+      if ((*fset)->farr[ftype] == NULL) continue;
+      tsdbTFileObjUnref((*fset)->farr[ftype]);
+    }
 
-  for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
-    if (fset[0]->farr[ftype] == NULL) continue;
-    tsdbTFileObjUnref(fset[0]->farr[ftype]);
+    TARRAY2_DESTROY((*fset)->lvlArr, tsdbSttLvlClear);
+
+    taosThreadCondDestroy(&(*fset)->beginTask);
+    taosThreadCondDestroy(&(*fset)->canCommit);
+    taosMemoryFreeClear(*fset);
   }
-
-  TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear);
-
-  taosThreadCondDestroy(&fset[0]->canCommit);
-  taosMemoryFree(fset[0]);
-  fset[0] = NULL;
-
-  return 0;
 }
 
 int32_t tsdbTFileSetRemove(STFileSet *fset) {
@@ -665,6 +664,12 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) {
 }
 
 int32_t tsdbTFileSetOpenChannel(STFileSet *fset) {
-  if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0;
-  return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel);
+  int32_t code;
+  if (!fset->channelOpened) {
+    if ((code = vnodeAChannelInit(2, &fset->channel))) {
+      return code;
+    }
+    fset->channelOpened = true;
+  }
+  return 0;
 }
diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h
index 86db3b01c7..4b99a46dc7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h
+++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h
@@ -42,7 +42,7 @@ typedef enum {
 int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset);
 int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
 int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
-int32_t tsdbTFileSetClear(STFileSet **fset);
+void    tsdbTFileSetClear(STFileSet **fset);
 int32_t tsdbTFileSetRemove(STFileSet *fset);
 
 int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
@@ -90,9 +90,15 @@ struct STFileSet {
   STFileObj   *farr[TSDB_FTYPE_MAX];  // file array
   TSttLvlArray lvlArr[1];             // level array
 
-  // background task channel
-  int64_t bgTaskChannel;
-  bool    mergeScheduled;
+  // background task
+  bool         channelOpened;
+  SVAChannelID channel;
+  bool         mergeScheduled;
+
+  // sttTrigger = 1
+  TdThreadCond beginTask;
+  bool         taskRunning;
+  int32_t      numWaitTask;
 
   // block commit variables
   TdThreadCond canCommit;
diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c
index 971020e7d6..022698b0eb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMerge.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c
@@ -593,5 +593,6 @@ _exit:
     exit(EXIT_FAILURE);
   }
   tsdbTFileSetClear(&merger->fset);
+  taosMemoryFree(arg);
   return code;
 }
diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c
index f4344296b4..3d53d1ada3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRetention.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@@ -24,8 +24,8 @@ typedef struct {
   int64_t now;
   int64_t cid;
 
-  TFileSetArray *fsetArr;
-  TFileOpArray   fopArr[1];
+  STFileSet   *fset;
+  TFileOpArray fopArr;
 } SRTNer;
 
 static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
@@ -35,7 +35,7 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
       .of = fobj->f[0],
   };
 
-  return TARRAY2_APPEND(rtner->fopArr, op);
+  return TARRAY2_APPEND(&rtner->fopArr, op);
 }
 
 static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
@@ -71,7 +71,7 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
     if (fdFrom) taosCloseFile(&fdFrom);
     if (fdTo) taosCloseFile(&fdTo);
   }
@@ -108,7 +108,7 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
     taosCloseFile(&fdFrom);
     taosCloseFile(&fdTo);
   }
@@ -128,7 +128,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
       .of = fobj->f[0],
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // create new
@@ -152,7 +152,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
           },
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // do copy the file
@@ -167,7 +167,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
   }
   return code;
 }
@@ -176,80 +176,51 @@ typedef struct {
   STsdb  *tsdb;
   int64_t now;
   int32_t fid;
+  bool    s3Migrate;
 } SRtnArg;
 
-static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  STsdb *tsdb = arg->tsdb;
-
-  rtner->tsdb = tsdb;
-  rtner->szPage = tsdb->pVnode->config.tsdbPageSize;
-  rtner->now = arg->now;
-  rtner->cid = tsdbFSAllocEid(tsdb->pFS);
-
-  code = tsdbFSCreateCopySnapshot(tsdb->pFS, &rtner->fsetArr);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
-  } else {
-    tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
-  }
-  return code;
-}
-
 static int32_t tsdbDoRetentionEnd(SRTNer *rtner) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  if (TARRAY2_SIZE(rtner->fopArr) == 0) goto _exit;
-
-  code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  taosThreadMutexLock(&rtner->tsdb->mutex);
-
-  code = tsdbFSEditCommit(rtner->tsdb->pFS);
-  if (code) {
-    taosThreadMutexUnlock(&rtner->tsdb->mutex);
+  if (TARRAY2_SIZE(&rtner->fopArr) > 0) {
+    code = tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION);
     TSDB_CHECK_CODE(code, lino, _exit);
+
+    taosThreadMutexLock(&rtner->tsdb->mutex);
+
+    code = tsdbFSEditCommit(rtner->tsdb->pFS);
+    if (code) {
+      taosThreadMutexUnlock(&rtner->tsdb->mutex);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+
+    taosThreadMutexUnlock(&rtner->tsdb->mutex);
+
+    TARRAY2_DESTROY(&rtner->fopArr, NULL);
   }
 
-  taosThreadMutexUnlock(&rtner->tsdb->mutex);
-
-  TARRAY2_DESTROY(rtner->fopArr, NULL);
-
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
   } else {
     tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
   }
-  tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
   return code;
 }
 
-static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
+static int32_t tsdbDoRetention(SRTNer *rtner) {
   int32_t    code = 0;
   int32_t    lino = 0;
   STFileObj *fobj = NULL;
+  STFileSet *fset = rtner->fset;
   int32_t    expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);
 
   if (expLevel < 0) {  // remove the fileset
     for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) {
       if (fobj == NULL) continue;
-      /*
-      int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
-      if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
-        code = tsdbRemoveFileObjectS3(rtner, fobj);
-        TSDB_CHECK_CODE(code, lino, _exit);
-        } else {*/
       code = tsdbDoRemoveFileObject(rtner, fobj);
       TSDB_CHECK_CODE(code, lino, _exit);
-      //}
     }
 
     SSttLvl *lvl;
@@ -275,10 +246,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
       if (fobj == NULL) continue;
 
       if (fobj->f->did.level == did.level) {
-        /*
-        code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did);
-        TSDB_CHECK_CODE(code, lino, _exit);
-        */
         continue;
       }
 
@@ -306,91 +273,110 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
   }
   return code;
 }
 
-static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
+static void tsdbRetentionCancel(void *arg) { taosMemoryFree(arg); }
 
-static int32_t tsdbDoRetentionAsync(void *arg) {
+static int32_t tsdbDoS3Migrate(SRTNer *rtner);
+
+static int32_t tsdbRetention(void *arg) {
   int32_t code = 0;
   int32_t lino = 0;
-  SRTNer  rtner[1] = {0};
 
-  code = tsdbDoRetentionBegin(arg, rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
+  SRtnArg   *rtnArg = (SRtnArg *)arg;
+  STsdb     *pTsdb = rtnArg->tsdb;
+  SVnode    *pVnode = pTsdb->pVnode;
+  STFileSet *fset = NULL;
+  SRTNer     rtner = {
+          .tsdb = pTsdb,
+          .szPage = pVnode->config.tsdbPageSize,
+          .now = rtnArg->now,
+          .cid = tsdbFSAllocEid(pTsdb->pFS),
+  };
 
-  STFileSet *fset;
-  TARRAY2_FOREACH(rtner->fsetArr, fset) {
-    if (fset->fid != ((SRtnArg *)arg)->fid) continue;
+  // begin task
+  taosThreadMutexLock(&pTsdb->mutex);
+  tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, &fset);
+  if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) {
+    taosThreadMutexUnlock(&pTsdb->mutex);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  taosThreadMutexUnlock(&pTsdb->mutex);
 
-    code = tsdbDoRetentionOnFileSet(rtner, fset);
+  // do retention
+  if (rtner.fset) {
+    if (rtnArg->s3Migrate) {
+      code = tsdbDoS3Migrate(&rtner);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    } else {
+      code = tsdbDoRetention(&rtner);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+
+    code = tsdbDoRetentionEnd(&rtner);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  code = tsdbDoRetentionEnd(rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
 _exit:
-  if (code) {
-    if (TARRAY2_DATA(rtner->fopArr)) {
-      TARRAY2_DESTROY(rtner->fopArr, NULL);
-    }
-    TFileSetArray **fsetArr = &rtner->fsetArr;
-    if (fsetArr[0]) {
-      tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
-    }
+  if (rtner.fset) {
+    taosThreadMutexLock(&pTsdb->mutex);
+    tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid);
+    taosThreadMutexUnlock(&pTsdb->mutex);
+  }
 
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+  // clear resources
+  tsdbTFileSetClear(&rtner.fset);
+  TARRAY2_DESTROY(&rtner.fopArr, NULL);
+  taosMemoryFree(arg);
+  if (code) {
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(((SRtnArg *)arg)->tsdb->pVnode), __func__, code, lino);
   }
   return code;
 }
 
-int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
+static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) {
   int32_t code = 0;
-
-  taosThreadMutexLock(&tsdb->mutex);
-
-  if (tsdb->bgTaskDisabled) {
-    taosThreadMutexUnlock(&tsdb->mutex);
-    return 0;
-  }
+  int32_t lino = 0;
 
   STFileSet *fset;
-  TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
-    code = tsdbTFileSetOpenChannel(fset);
-    if (code) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
-    }
 
-    SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
-    if (arg == NULL) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
+  if (!tsdb->bgTaskDisabled) {
+    TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
+      code = tsdbTFileSetOpenChannel(fset);
+      TSDB_CHECK_CODE(code, lino, _exit);
 
-    arg->tsdb = tsdb;
-    arg->now = now;
-    arg->fid = fset->fid;
+      SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
+      if (arg == NULL) {
+        TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+      }
 
-    if (sync) {
-      code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync,
-                         tsdbFreeRtnArg, arg, NULL);
-    } else {
-      code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync,
-                         tsdbFreeRtnArg, arg, NULL);
-    }
-    if (code) {
-      tsdbFreeRtnArg(arg);
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
+      arg->tsdb = tsdb;
+      arg->now = now;
+      arg->fid = fset->fid;
+      arg->s3Migrate = s3Migrate;
+
+      if ((code = vnodeAsync(&fset->channel, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg, NULL))) {
+        taosMemoryFree(arg);
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
     }
   }
 
-  taosThreadMutexUnlock(&tsdb->mutex);
+_exit:
+  if (code) {
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(tsdb->pVnode), __func__, code, lino);
+  }
+  return code;
+}
 
+int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now) {
+  int32_t code = 0;
+  taosThreadMutexLock(&tsdb->mutex);
+  code = tsdbAsyncRetentionImpl(tsdb, now, false);
+  taosThreadMutexUnlock(&tsdb->mutex);
   return code;
 }
 
@@ -462,7 +448,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
       .of = fobj->f[0],
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // create new
@@ -486,7 +472,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
           },
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   char fname[TSDB_FILENAME_LEN];
@@ -566,7 +552,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
       .of = fobj->f[0],
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // create new
@@ -590,7 +576,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
           },
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   char fname[TSDB_FILENAME_LEN];
@@ -653,10 +639,11 @@ _exit:
   return code;
 }
 
-static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
+static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
   int32_t code = 0;
   int32_t lino = 0;
 
+  STFileSet *fset = rtner->fset;
   STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
   if (!fobj) return code;
 
@@ -720,41 +707,7 @@ _exit:
   return code;
 }
 
-static int32_t tsdbDoS3MigrateAsync(void *arg) {
-  int32_t code = 0;
-  int32_t lino = 0;
-  SRTNer  rtner[1] = {0};
-
-  code = tsdbDoRetentionBegin(arg, rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  STFileSet *fset;
-  TARRAY2_FOREACH(rtner->fsetArr, fset) {
-    if (fset->fid != ((SRtnArg *)arg)->fid) continue;
-
-    code = tsdbDoS3MigrateOnFileSet(rtner, fset);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  code = tsdbDoRetentionEnd(rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    if (TARRAY2_DATA(rtner->fopArr)) {
-      TARRAY2_DESTROY(rtner->fopArr, NULL);
-    }
-    TFileSetArray **fsetArr = &rtner->fsetArr;
-    if (fsetArr[0]) {
-      tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
-    }
-
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
-  }
-  return code;
-}
-
-int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
+int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
   int32_t code = 0;
 
   extern int8_t tsS3EnabledCfg;
@@ -772,45 +725,7 @@ int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
   }
 
   taosThreadMutexLock(&tsdb->mutex);
-
-  if (tsdb->bgTaskDisabled) {
-    taosThreadMutexUnlock(&tsdb->mutex);
-    return 0;
-  }
-
-  STFileSet *fset;
-  TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
-    code = tsdbTFileSetOpenChannel(fset);
-    if (code) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
-    }
-
-    SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
-    if (arg == NULL) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    arg->tsdb = tsdb;
-    arg->now = now;
-    arg->fid = fset->fid;
-
-    if (sync) {
-      code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
-                         tsdbFreeRtnArg, arg, NULL);
-    } else {
-      code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
-                         tsdbFreeRtnArg, arg, NULL);
-    }
-    if (code) {
-      tsdbFreeRtnArg(arg);
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
-    }
-  }
-
+  code = tsdbAsyncRetentionImpl(tsdb, now, true);
   taosThreadMutexUnlock(&tsdb->mutex);
-
   return code;
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c
index 313603b19f..6d6533463b 100644
--- a/source/dnode/vnode/src/vnd/vnodeAsync.c
+++ b/source/dnode/vnode/src/vnd/vnodeAsync.c
@@ -16,6 +16,7 @@
 #include "vnd.h"
 #include "vnodeHash.h"
 
+typedef struct SVAsync    SVAsync;
 typedef struct SVATask    SVATask;
 typedef struct SVAChannel SVAChannel;
 
@@ -54,7 +55,7 @@ struct SVATask {
   int32_t     priorScore;
   SVAChannel *channel;
   int32_t (*execute)(void *);
-  void (*complete)(void *);
+  void (*cancel)(void *);
   void        *arg;
   EVATaskState state;
 
@@ -67,6 +68,11 @@ struct SVATask {
   struct SVATask *next;
 };
 
+typedef struct {
+  void (*cancel)(void *);
+  void *arg;
+} SVATaskCancelInfo;
+
 #define VATASK_PIORITY(task_) ((task_)->priority - ((task_)->priorScore / 4))
 
 // async channel
@@ -112,6 +118,10 @@ struct SVAsync {
   SVHashTable *taskTable;
 };
 
+SVAsync *vnodeAsyncs[3];
+#define MIN_ASYNC_ID 1
+#define MAX_ASYNC_ID (sizeof(vnodeAsyncs) / sizeof(vnodeAsyncs[0]) - 1)
+
 static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
   int32_t ret;
 
@@ -160,11 +170,6 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
   }
   async->numTasks--;
 
-  // call complete callback
-  if (task->complete) {
-    task->complete(task->arg);
-  }
-
   if (task->numWait == 0) {
     taosThreadCondDestroy(&task->waitCond);
     taosMemoryFree(task);
@@ -176,7 +181,7 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
   return 0;
 }
 
-static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
+static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) {
   while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] ||
          async->queue[2].next != &async->queue[2]) {
     for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
@@ -184,6 +189,12 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
         SVATask *task = async->queue[i].next;
         task->prev->next = task->next;
         task->next->prev = task->prev;
+        if (task->cancel) {
+          taosArrayPush(cancelArray, &(SVATaskCancelInfo){
+                                         .cancel = task->cancel,
+                                         .arg = task->arg,
+                                     });
+        }
         vnodeAsyncTaskDone(async, task);
       }
     }
@@ -194,6 +205,7 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
 static void *vnodeAsyncLoop(void *arg) {
   SVWorker *worker = (SVWorker *)arg;
   SVAsync  *async = worker->async;
+  SArray   *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
 
   setThreadName(async->label);
 
@@ -209,12 +221,12 @@ static void *vnodeAsyncLoop(void *arg) {
     for (;;) {
       if (async->stop || worker->workerId >= async->numWorkers) {
         if (async->stop) {  // cancel all tasks
-          vnodeAsyncCancelAllTasks(async);
+          vnodeAsyncCancelAllTasks(async, cancelArray);
         }
         worker->state = EVA_WORKER_STATE_STOP;
         async->numLaunchWorkers--;
         taosThreadMutexUnlock(&async->mutex);
-        return NULL;
+        goto _exit;
       }
 
       for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
@@ -259,6 +271,12 @@ static void *vnodeAsyncLoop(void *arg) {
     worker->runningTask->execute(worker->runningTask->arg);
   }
 
+_exit:
+  for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) {
+    SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i);
+    cancel->cancel(cancel->arg);
+  }
+  taosArrayDestroy(cancelArray);
   return NULL;
 }
 
@@ -294,7 +312,7 @@ static int32_t vnodeAsyncChannelCompare(const void *obj1, const void *obj2) {
   return 0;
 }
 
-int32_t vnodeAsyncInit(SVAsync **async, char *label) {
+static int32_t vnodeAsyncInit(SVAsync **async, const char *label) {
   int32_t ret;
 
   if (async == NULL) {
@@ -360,7 +378,7 @@ int32_t vnodeAsyncInit(SVAsync **async, char *label) {
   return 0;
 }
 
-int32_t vnodeAsyncDestroy(SVAsync **async) {
+static int32_t vnodeAsyncDestroy(SVAsync **async) {
   if ((*async) == NULL) {
     return TSDB_CODE_INVALID_PARA;
   }
@@ -433,20 +451,39 @@ static int32_t vnodeAsyncLaunchWorker(SVAsync *async) {
   return 0;
 }
 
-#ifdef BUILD_NO_CALL
-int32_t vnodeAsync(SVAsync *async, EVAPriority priority, int32_t (*execute)(void *), void (*complete)(void *),
-                   void *arg, int64_t *taskId) {
-  return vnodeAsyncC(async, 0, priority, execute, complete, arg, taskId);
-}
-#endif
+int32_t vnodeAsyncOpen(int32_t numOfThreads) {
+  int32_t code = 0;
+  int32_t lino = 0;
 
-int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void *),
-                    void (*complete)(void *), void *arg, int64_t *taskId) {
-  if (async == NULL || execute == NULL || channelId < 0) {
+  // vnode-commit
+  code = vnodeAsyncInit(&vnodeAsyncs[1], "vnode-commit");
+  TSDB_CHECK_CODE(code, lino, _exit);
+  vnodeAsyncSetWorkers(1, numOfThreads);
+
+  // vnode-merge
+  code = vnodeAsyncInit(&vnodeAsyncs[2], "vnode-merge");
+  TSDB_CHECK_CODE(code, lino, _exit);
+  vnodeAsyncSetWorkers(2, numOfThreads);
+
+_exit:
+  return 0;
+}
+
+int32_t vnodeAsyncClose() {
+  vnodeAsyncDestroy(&vnodeAsyncs[1]);
+  vnodeAsyncDestroy(&vnodeAsyncs[2]);
+  return 0;
+}
+
+int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*execute)(void *), void (*cancel)(void *),
+                   void *arg, SVATaskID *taskID) {
+  if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || execute == NULL ||
+      channelID->id < 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
-  int64_t id;
+  int64_t  id;
+  SVAsync *async = vnodeAsyncs[channelID->async];
 
   // create task object
   SVATask *task = (SVATask *)taosMemoryCalloc(1, sizeof(SVATask));
@@ -457,7 +494,7 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
   task->priority = priority;
   task->priorScore = 0;
   task->execute = execute;
-  task->complete = complete;
+  task->cancel = cancel;
   task->arg = arg;
   task->state = EVA_TASK_STATE_WAITTING;
   task->numWait = 0;
@@ -466,10 +503,12 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
   // schedule task
   taosThreadMutexLock(&async->mutex);
 
-  if (channelId == 0) {
+  if (channelID->id == 0) {
     task->channel = NULL;
   } else {
-    SVAChannel channel = {.channelId = channelId};
+    SVAChannel channel = {
+        .channelId = channelID->id,
+    };
     vHashGet(async->channelTable, &channel, (void **)&task->channel);
     if (task->channel == NULL) {
       taosThreadMutexUnlock(&async->mutex);
@@ -540,20 +579,24 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
 
   taosThreadMutexUnlock(&async->mutex);
 
-  if (taskId != NULL) {
-    *taskId = id;
+  if (taskID != NULL) {
+    taskID->async = channelID->async;
+    taskID->id = id;
   }
 
   return 0;
 }
 
-int32_t vnodeAWait(SVAsync *async, int64_t taskId) {
-  if (async == NULL || taskId <= 0) {
+int32_t vnodeAWait(SVATaskID *taskID) {
+  if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
+  SVAsync *async = vnodeAsyncs[taskID->async];
   SVATask *task = NULL;
-  SVATask  task2 = {.taskId = taskId};
+  SVATask  task2 = {
+       .taskId = taskID->id,
+  };
 
   taosThreadMutexLock(&async->mutex);
 
@@ -574,21 +617,27 @@ int32_t vnodeAWait(SVAsync *async, int64_t taskId) {
   return 0;
 }
 
-int32_t vnodeACancel(SVAsync *async, int64_t taskId) {
-  if (async == NULL) {
+int32_t vnodeACancel(SVATaskID *taskID) {
+  if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
   int32_t  ret = 0;
+  SVAsync *async = vnodeAsyncs[taskID->async];
   SVATask *task = NULL;
-  SVATask  task2 = {.taskId = taskId};
+  SVATask  task2 = {
+       .taskId = taskID->id,
+  };
+  void (*cancel)(void *) = NULL;
+  void *arg = NULL;
 
   taosThreadMutexLock(&async->mutex);
 
   vHashGet(async->taskTable, &task2, (void **)&task);
   if (task) {
     if (task->state == EVA_TASK_STATE_WAITTING) {
-      // remove from queue
+      cancel = task->cancel;
+      arg = task->arg;
       task->next->prev = task->prev;
       task->prev->next = task->next;
       vnodeAsyncTaskDone(async, task);
@@ -599,14 +648,18 @@ int32_t vnodeACancel(SVAsync *async, int64_t taskId) {
 
   taosThreadMutexUnlock(&async->mutex);
 
+  if (cancel) {
+    cancel(arg);
+  }
+
   return ret;
 }
 
-int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) {
-  if (async == NULL || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) {
+int32_t vnodeAsyncSetWorkers(int64_t asyncID, int32_t numWorkers) {
+  if (asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) {
     return TSDB_CODE_INVALID_PARA;
   }
-
+  SVAsync *async = vnodeAsyncs[asyncID];
   taosThreadMutexLock(&async->mutex);
   async->numWorkers = numWorkers;
   if (async->numIdleWorkers > 0) {
@@ -617,11 +670,13 @@ int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) {
   return 0;
 }
 
-int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
-  if (async == NULL || channelId == NULL) {
+int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) {
+  if (channelID == NULL || asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID) {
     return TSDB_CODE_INVALID_PARA;
   }
 
+  SVAsync *async = vnodeAsyncs[asyncID];
+
   // create channel object
   SVAChannel *channel = (SVAChannel *)taosMemoryMalloc(sizeof(SVAChannel));
   if (channel == NULL) {
@@ -637,7 +692,7 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
   // register channel
   taosThreadMutexLock(&async->mutex);
 
-  channel->channelId = *channelId = ++async->nextChannelId;
+  channel->channelId = channelID->id = ++async->nextChannelId;
 
   // add to hash table
   int32_t ret = vHashPut(async->channelTable, channel);
@@ -657,16 +712,24 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
 
   taosThreadMutexUnlock(&async->mutex);
 
+  channelID->async = asyncID;
   return 0;
 }
 
-int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning) {
-  if (async == NULL || channelId <= 0) {
+int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
+  if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || channelID->id <= 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
+  SVAsync    *async = vnodeAsyncs[channelID->async];
   SVAChannel *channel = NULL;
-  SVAChannel  channel2 = {.channelId = channelId};
+  SVAChannel  channel2 = {
+       .channelId = channelID->id,
+  };
+  SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
+  if (cancelArray == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
 
   taosThreadMutexLock(&async->mutex);
 
@@ -684,6 +747,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
         SVATask *task = channel->queue[i].next;
         task->prev->next = task->next;
         task->next->prev = task->prev;
+        if (task->cancel) {
+          taosArrayPush(cancelArray, &(SVATaskCancelInfo){
+                                         .cancel = task->cancel,
+                                         .arg = task->arg,
+                                     });
+        }
         vnodeAsyncTaskDone(async, task);
       }
     }
@@ -693,6 +762,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
       if (channel->scheduled) {
         channel->scheduled->prev->next = channel->scheduled->next;
         channel->scheduled->next->prev = channel->scheduled->prev;
+        if (channel->scheduled->cancel) {
+          taosArrayPush(cancelArray, &(SVATaskCancelInfo){
+                                         .cancel = channel->scheduled->cancel,
+                                         .arg = channel->scheduled->arg,
+                                     });
+        }
         vnodeAsyncTaskDone(async, channel->scheduled);
       }
       taosMemoryFree(channel);
@@ -713,12 +788,16 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
         channel->state = EVA_CHANNEL_STATE_CLOSE;
       }
     }
-  } else {
-    taosThreadMutexUnlock(&async->mutex);
-    return TSDB_CODE_INVALID_PARA;
   }
 
   taosThreadMutexUnlock(&async->mutex);
+  for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) {
+    SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i);
+    cancel->cancel(cancel->arg);
+  }
+  taosArrayDestroy(cancelArray);
 
+  channelID->async = 0;
+  channelID->id = 0;
   return 0;
 }
\ No newline at end of file
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 645f2620dc..f071775990 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -289,7 +289,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
   int64_t lastCommitted = pInfo->info.state.committed;
 
   // wait last commit task
-  vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
+  vnodeAWait(&pVnode->commitTask);
 
   if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit;
 
@@ -361,15 +361,14 @@ static void vnodeReturnBufPool(SVnode *pVnode) {
 
   taosThreadMutexUnlock(&pVnode->mutex);
 }
-static int32_t vnodeCommitTask(void *arg) {
+static int32_t vnodeCommit(void *arg) {
   int32_t code = 0;
 
   SCommitInfo *pInfo = (SCommitInfo *)arg;
   SVnode      *pVnode = pInfo->pVnode;
 
   // commit
-  code = vnodeCommitImpl(pInfo);
-  if (code) {
+  if ((code = vnodeCommitImpl(pInfo))) {
     vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr());
     taosMsleep(100);
     exit(EXIT_FAILURE);
@@ -379,37 +378,34 @@ static int32_t vnodeCommitTask(void *arg) {
   vnodeReturnBufPool(pVnode);
 
 _exit:
+  taosMemoryFree(arg);
   return code;
 }
 
-static void vnodeCompleteCommit(void *arg) { taosMemoryFree(arg); }
+static void vnodeCommitCancel(void *arg) { taosMemoryFree(arg); }
 
 int vnodeAsyncCommit(SVnode *pVnode) {
   int32_t code = 0;
+  int32_t lino = 0;
 
   SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo));
   if (NULL == pInfo) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
   }
 
   // prepare to commit
   code = vnodePrepareCommit(pVnode, pInfo);
-  if (TSDB_CODE_SUCCESS != code) {
-    goto _exit;
-  }
+  TSDB_CHECK_CODE(code, lino, _exit);
 
   // schedule the task
-  code = vnodeAsyncC(vnodeAsyncHandle[0], pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommitTask,
-                     vnodeCompleteCommit, pInfo, &pVnode->commitTask);
+  code =
+      vnodeAsync(&pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommit, vnodeCommitCancel, pInfo, &pVnode->commitTask);
+  TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
   if (code) {
-    if (NULL != pInfo) {
-      taosMemoryFree(pInfo);
-    }
-    vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code),
-           pVnode->state.commitID);
+    taosMemoryFree(pInfo);
+    vError("vgId:%d %s failed at line %d since %s" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code));
   } else {
     vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode),
           pVnode->state.commitID, pVnode->state.applyTerm, pVnode->state.applied);
@@ -419,7 +415,7 @@ _exit:
 
 int vnodeSyncCommit(SVnode *pVnode) {
   vnodeAsyncCommit(pVnode);
-  vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
+  vnodeAWait(&pVnode->commitTask);
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeHash.h b/source/dnode/vnode/src/vnd/vnodeHash.h
index 0181ca748d..00b3488930 100644
--- a/source/dnode/vnode/src/vnd/vnodeHash.h
+++ b/source/dnode/vnode/src/vnd/vnodeHash.h
@@ -22,23 +22,6 @@
 extern "C" {
 #endif
 
-typedef struct SVHashTable SVHashTable;
-
-struct SVHashTable {
-  uint32_t (*hash)(const void*);
-  int32_t (*compare)(const void*, const void*);
-  int32_t              numEntries;
-  uint32_t             numBuckets;
-  struct SVHashEntry** buckets;
-};
-
-#define vHashNumEntries(ht) ((ht)->numEntries)
-int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*));
-int32_t vHashDestroy(SVHashTable** ht);
-int32_t vHashPut(SVHashTable* ht, void* obj);
-int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj);
-int32_t vHashDrop(SVHashTable* ht, const void* obj);
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c
index 44fcbefba7..3a454c53ef 100644
--- a/source/dnode/vnode/src/vnd/vnodeModule.c
+++ b/source/dnode/vnode/src/vnd/vnodeModule.c
@@ -18,8 +18,6 @@
 
 static volatile int32_t VINIT = 0;
 
-SVAsync* vnodeAsyncHandle[2];
-
 int vnodeInit(int nthreads) {
   int32_t init;
 
@@ -28,13 +26,9 @@ int vnodeInit(int nthreads) {
     return 0;
   }
 
-  // vnode-commit
-  vnodeAsyncInit(&vnodeAsyncHandle[0], "vnode-commit");
-  vnodeAsyncSetWorkers(vnodeAsyncHandle[0], nthreads);
-
-  // vnode-merge
-  vnodeAsyncInit(&vnodeAsyncHandle[1], "vnode-merge");
-  vnodeAsyncSetWorkers(vnodeAsyncHandle[1], nthreads);
+  if (vnodeAsyncOpen(nthreads) != 0) {
+    return -1;
+  }
 
   if (walInit() < 0) {
     return -1;
@@ -48,8 +42,7 @@ void vnodeCleanup() {
   if (init == 0) return;
 
   // set stop
-  vnodeAsyncDestroy(&vnodeAsyncHandle[0]);
-  vnodeAsyncDestroy(&vnodeAsyncHandle[1]);
+  vnodeAsyncClose();
 
   walCleanUp();
   smaCleanUp();
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 425cff7ce9..da8c3a6cad 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -399,7 +399,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
   taosThreadMutexInit(&pVnode->mutex, NULL);
   taosThreadCondInit(&pVnode->poolNotEmpty, NULL);
 
-  if (vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel) != 0) {
+  if (vnodeAChannelInit(1, &pVnode->commitChannel) != 0) {
     vError("vgId:%d, failed to init commit channel", TD_VID(pVnode));
     goto _err;
   }
@@ -527,8 +527,8 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); }
 
 void vnodeClose(SVnode *pVnode) {
   if (pVnode) {
-    vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
-    vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true);
+    vnodeAWait(&pVnode->commitTask);
+    vnodeAChannelDestroy(&pVnode->commitChannel, true);
     vnodeSyncClose(pVnode);
     vnodeQueryClose(pVnode);
     tqClose(pVnode->pTq);
diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c
index 5db20b8fc7..6dca7a9a60 100644
--- a/source/dnode/vnode/src/vnd/vnodeRetention.c
+++ b/source/dnode/vnode/src/vnd/vnodeRetention.c
@@ -15,20 +15,15 @@
 
 #include "vnd.h"
 
-int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
-  int32_t code = TSDB_CODE_SUCCESS;
+extern int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now);
+extern int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now);
 
-  code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
-
-  if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
-
-  return code;
+int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now) {
+  // async retention
+  return tsdbAsyncRetention(pVnode->pTsdb, now);
 }
 
-int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
-  code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
-
-  return code;
+int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now) {
+  // async migration
+  return tsdbAsyncS3Migrate(pVnode->pTsdb, now);
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index ea742108aa..4bb5fbc822 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -622,13 +622,13 @@ extern int32_t tsdbEnableBgTask(STsdb *pTsdb);
 static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) {
   tsdbDisableAndCancelAllBgTask(pVnode->pTsdb);
   vnodeSyncCommit(pVnode);
-  vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true);
+  vnodeAChannelDestroy(&pVnode->commitChannel, true);
   return 0;
 }
 
 static int32_t vnodeEnableBgTask(SVnode *pVnode) {
   tsdbEnableBgTask(pVnode->pTsdb);
-  vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel);
+  vnodeAChannelInit(1, &pVnode->commitChannel);
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index cb71495e9f..388bc6aef3 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -50,7 +50,7 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l
 
 static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
 static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
-static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp);
+static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 
 extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg);
@@ -866,7 +866,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) {
   pMetaRsp->precision = pVnode->config.tsdbCfg.precision;
 }
 
-extern int32_t vnodeDoRetention(SVnode *pVnode, int64_t now);
+extern int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now);
 
 static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t     code = 0;
@@ -880,13 +880,13 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int3
 
   vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp);
 
-  code = vnodeDoRetention(pVnode, trimReq.timestamp);
+  code = vnodeAsyncRetention(pVnode, trimReq.timestamp);
 
 _exit:
   return code;
 }
 
-extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now);
+extern int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now);
 
 static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t          code = 0;
@@ -900,7 +900,7 @@ static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq,
 
   vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp);
 
-  code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp);
+  code = vnodeAsyncS3Migrate(pVnode, s3migrateReq.timestamp);
 
 _exit:
   return code;
@@ -931,13 +931,13 @@ end:
   return ret;
 }
 
-static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp) {
+static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t                 code = -1;
   SMetaReader             mr = {0};
   SVDropTtlTableReq       ttlReq = {0};
   SVFetchTtlExpiredTbsRsp rsp = {0};
   SEncoder                encoder = {0};
-  SArray*                 pNames = NULL;
+  SArray                 *pNames = NULL;
   pRsp->msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP;
   pRsp->code = TSDB_CODE_SUCCESS;
   pRsp->pCont = NULL;
@@ -950,8 +950,8 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void*
 
   ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids));
 
-  tb_uid_t       suid;
-  char           ctbName[TSDB_TABLE_NAME_LEN];
+  tb_uid_t    suid;
+  char        ctbName[TSDB_TABLE_NAME_LEN];
   SVDropTbReq expiredTb = {.igNotExists = true};
   metaReaderDoInit(&mr, pVnode->pMeta, 0);
   rsp.vgId = TD_VID(pVnode);
@@ -965,12 +965,12 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void*
   }
   char buf[TSDB_TABLE_NAME_LEN];
   for (int32_t i = 0; i < ttlReq.nUids; ++i) {
-    tb_uid_t* uid = taosArrayGet(ttlReq.pTbUids, i);
+    tb_uid_t *uid = taosArrayGet(ttlReq.pTbUids, i);
     expiredTb.suid = *uid;
     terrno = metaReaderGetTableEntryByUid(&mr, *uid);
     if (terrno < 0) goto _end;
     strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN);
-    void* p = taosArrayPush(pNames, buf);
+    void *p = taosArrayPush(pNames, buf);
     expiredTb.name = p;
     if (mr.me.type == TSDB_CHILD_TABLE) {
       expiredTb.suid = mr.me.ctbEntry.suid;
@@ -1144,7 +1144,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
       if (i < tbNames->size - 1) {
         taosStringBuilderAppendChar(&sb, ',');
       }
-      //taosMemoryFreeClear(*key);
+      // taosMemoryFreeClear(*key);
     }
 
     size_t len = 0;
@@ -2241,14 +2241,14 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq,
   return TSDB_CODE_SUCCESS;
 }
 
-extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+extern int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 
 static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   if (!pVnode->restored) {
     vInfo("vgId:%d, ignore compact req during restoring. ver:%" PRId64, TD_VID(pVnode), ver);
     return 0;
   }
-  return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp);
+  return vnodeAsyncCompact(pVnode, ver, pReq, len, pRsp);
 }
 
 static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c
index f2b45c5b84..e7c4573c14 100644
--- a/source/libs/tfs/src/tfs.c
+++ b/source/libs/tfs/src/tfs.c
@@ -177,11 +177,10 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
       continue;
     }
 
-    return 0;
+    return (terrno = 0);
   }
 
-  terrno = TSDB_CODE_FS_NO_VALID_DISK;
-  return -1;
+  return (terrno = TSDB_CODE_FS_NO_VALID_DISK);
 }
 
 const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; }

From 21ea1e3735943707220dbd9a125b257fd30f6040 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 23 May 2024 19:05:11 +0800
Subject: [PATCH 032/132] fix memory leak

---
 source/dnode/vnode/src/tsdb/tsdbCommit2.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index ff683a12ba..1a08f2dc82 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -38,12 +38,13 @@ typedef struct {
   struct {
     SFileSetCommitInfo *info;
 
-    int32_t   expLevel;
-    SDiskID   did;
-    TSKEY     minKey;
-    TSKEY     maxKey;
-    TABLEID   tbid[1];
-    bool      hasTSData;
+    int32_t expLevel;
+    SDiskID did;
+    TSKEY   minKey;
+    TSKEY   maxKey;
+    TABLEID tbid[1];
+    bool    hasTSData;
+
     bool      skipTsRow;
     SHashObj *pColCmprObj;
   } ctx[1];
@@ -116,7 +117,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
 
   committer->ctx->tbid->suid = 0;
   committer->ctx->tbid->uid = 0;
-
   for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) {
     if (row->uid != committer->ctx->tbid->uid) {
       committer->ctx->tbid->suid = row->suid;
@@ -284,7 +284,7 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
   config.from->version = VERSION_MIN;
   config.from->key = (SRowKey){
       .ts = committer->ctx->minKey,
-      .numOfPKs = 0,  // TODO: support multiple primary keys
+      .numOfPKs = 0,
   };
 
   code = tsdbIterOpen(&config, &iter);
@@ -760,7 +760,9 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
 
   if (tsdb->imem) {
     SMemTable *pMemTable = tsdb->imem;
+
     taosThreadMutexLock(&tsdb->mutex);
+
     if ((code = tsdbFSEditCommit(tsdb->pFS))) {
       taosThreadMutexUnlock(&tsdb->mutex);
       TSDB_CHECK_CODE(code, lino, _exit);
@@ -775,6 +777,8 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
     }
 
     taosThreadMutexUnlock(&tsdb->mutex);
+
+    tsdbCommitInfoDestroy(tsdb);
     tsdbUnrefMemTable(pMemTable, NULL, true);
   }
 
@@ -813,4 +817,4 @@ _exit:
     tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__);
   }
   return code;
-}
+}
\ No newline at end of file

From 6064ea01252490131d1a2224e2373f65098d5aa4 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 24 May 2024 11:24:19 +0800
Subject: [PATCH 033/132] fix: support timeline query from union and order by

---
 include/libs/nodes/querynodes.h         |  2 +
 source/libs/nodes/src/nodesCloneFuncs.c |  3 +
 source/libs/parser/src/parTranslater.c  | 91 +++++++++++++++++++------
 3 files changed, 77 insertions(+), 19 deletions(-)

diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h
index 5b889171b3..befd6afce9 100644
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@@ -415,6 +415,7 @@ typedef struct SSelectStmt {
   int32_t       returnRows;  // EFuncReturnRows
   ETimeLineMode timeLineCurMode;
   ETimeLineMode timeLineResMode;
+  bool          timeLineFromOrderBy;
   bool          isEmptyResult;
   bool          isSubquery;
   bool          hasAggFuncs;
@@ -453,6 +454,7 @@ typedef struct SSetOperator {
   char             stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t          precision;
   ETimeLineMode    timeLineResMode;  
+  bool             timeLineFromOrderBy;
   bool             joinContains;
 } SSetOperator;
 
diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c
index 1a51620856..84d3f734fe 100644
--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@@ -845,6 +845,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
   COPY_SCALAR_FIELD(precision);
   COPY_SCALAR_FIELD(isEmptyResult);
   COPY_SCALAR_FIELD(timeLineResMode);
+  COPY_SCALAR_FIELD(timeLineFromOrderBy);
   COPY_SCALAR_FIELD(timeLineCurMode);
   COPY_SCALAR_FIELD(hasAggFuncs);
   COPY_SCALAR_FIELD(hasRepeatScanFuncs);
@@ -862,6 +863,8 @@ static int32_t setOperatorCopy(const SSetOperator* pSrc, SSetOperator* pDst) {
   COPY_CHAR_ARRAY_FIELD(stmtName);
   COPY_SCALAR_FIELD(precision);
   COPY_SCALAR_FIELD(timeLineResMode);
+  COPY_SCALAR_FIELD(timeLineFromOrderBy);
+  
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 79b37c2950..b326224793 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -959,7 +959,8 @@ static bool isTimeLineQuery(SNode* pStmt) {
     return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineCurMode) ||
            (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineCurMode);
   } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) {
-    return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode;
+    return (TIME_LINE_MULTI == ((SSetOperator*)pStmt)->timeLineResMode) ||
+           (TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode);
   } else {
     return false;
   }
@@ -1000,18 +1001,56 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) {
   return false;
 }
 
+SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList) {
+  SNodeList* pPartitionList = NULL;
+  SNode* pNode = NULL;
+  FOREACH(pNode, pOrderList) {
+    if (pNode == pOrderList->pTail->pNode) {
+      break;
+    }
+    SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode;
+    nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr));
+  }
+
+  return pPartitionList;
+}
+
+
 static bool isTimeLineAlignedQuery(SNode* pStmt) {
   SSelectStmt* pSelect = (SSelectStmt*)pStmt;
   if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
     return false;
   }
-  if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
-    return false;
+  if (QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
+    SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
+    if (pSelect->pPartitionByList) {
+      if (!pSub->timeLineFromOrderBy && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
+        return true;
+      }
+      if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
+        SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList);
+        bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
+        nodesDestroyList(pPartitionList);
+
+        if (match) {
+          return true;
+        }
+      }
+    }
   }
-  SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
-  if (pSelect->pPartitionByList && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
-    return true;
+  if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
+    SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
+    if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
+      SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList);
+      bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
+      nodesDestroyList(pPartitionList);
+
+      if (match) {
+        return true;
+      }
+    }
   }
+
   return false;
 }
 
@@ -6025,9 +6064,18 @@ static void resetResultTimeline(SSelectStmt* pSelect) {
   if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) ||
       (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) {
     pSelect->timeLineResMode = TIME_LINE_GLOBAL;
-  } else {
-    pSelect->timeLineResMode = TIME_LINE_NONE;
+    return;
+  } else if (pSelect->pOrderByList->length > 1) {
+    pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, pSelect->pOrderByList->length - 1))->pExpr;
+    if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) ||
+        (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) {
+      pSelect->timeLineResMode = TIME_LINE_MULTI;
+      pSelect->timeLineFromOrderBy = true;
+      return;
+    }
   }
+  
+  pSelect->timeLineResMode = TIME_LINE_NONE;
 }
 
 static int32_t replaceOrderByAliasForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
@@ -6180,16 +6228,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
     }
     snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
     SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
-    if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)) {
-      SColumnNode* pLCol = (SColumnNode*)pLeft;
-      SColumnNode* pRCol = (SColumnNode*)pRight;
+    bool isLeftPrimTs = isPrimaryKeyImpl(pLeft);
+    bool isRightPrimTs = isPrimaryKeyImpl(pRight);
+
+    if (isLeftPrimTs && isRightPrimTs) {
       SColumnNode* pFCol = (SColumnNode*)pProj;
-      if (pLCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pRCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
-        if (pLCol->isPrimTs && pRCol->isPrimTs) {
-          pFCol->isPrimTs = true;
-        }
-      }
+      pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
+      pFCol->isPrimTs = true;
     }
     if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) {
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -6225,9 +6270,17 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
     SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr;
     if (isPrimaryKeyImpl(pOrder)) {
       pSetOperator->timeLineResMode = TIME_LINE_GLOBAL;
-    } else {
-      pSetOperator->timeLineResMode = TIME_LINE_NONE;
+      return code;
+    } else if (pSetOperator->pOrderByList->length > 1) {
+      pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, pSetOperator->pOrderByList->length - 1))->pExpr;
+      if (isPrimaryKeyImpl(pOrder)) {
+        pSetOperator->timeLineResMode = TIME_LINE_MULTI;
+        pSetOperator->timeLineFromOrderBy = true;
+        return code;
+      }
     }
+    
+    pSetOperator->timeLineResMode = TIME_LINE_NONE;
   }
   return code;
 }

From be82e4acd2b2350c1a01f7ae2b7f5cc94cfd64b0 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 24 May 2024 11:25:30 +0800
Subject: [PATCH 034/132] add log

---
 source/client/src/clientTmq.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index df08a66bd3..1d993eef6f 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1439,6 +1439,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     }
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead));
+    tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId,
+             requestId);
   } else {  // invalid rspType
     tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
   }
@@ -1648,6 +1650,7 @@ SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper)
   pRspObj->common.vgId = pWrapper->vgHandle->vgId;
 
   memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp));
+  tscDebug("build batchmeta Rsp from wrapper");
   return pRspObj;
 }
 

From d33b2521fd87f454b084c96925cebb928ba14a90 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 24 May 2024 11:42:08 +0800
Subject: [PATCH 035/132] adj res type

---
 source/client/src/clientTmq.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 1d993eef6f..3fbae1b774 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -2260,7 +2260,7 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
   } else if (TD_RES_TMQ_METADATA(res)) {
     return TMQ_RES_METADATA;
   } else if (TD_RES_TMQ_BATCH_META(res)) {
-    return TMQ_RES_BATCH_TABLE_META;
+    return TMQ_RES_TABLE_META;
   } else {
     return TMQ_RES_INVALID;
   }

From 9947d8ec42af3d6e55d1b21bdafa769b28e7a133 Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Fri, 24 May 2024 12:57:49 +0800
Subject: [PATCH 036/132] fix: last cache memory leak

---
 source/dnode/vnode/src/tsdb/tsdbCache.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index d0c753d280..42df7f111e 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -588,6 +588,13 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud
     tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud);
   }
 
+  for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) {
+    SValue *pValue = &pLastCol->rowKey.pks[i];
+    if (IS_VAR_DATA_TYPE(pValue->type)) {
+      taosMemoryFree(pValue->pData);
+    }
+  }
+
   if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) /* && pLastCol->colVal.value.nData > 0*/) {
     taosMemoryFree(pLastCol->colVal.value.pData);
   }
@@ -1072,6 +1079,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
       SLastCol *PToFree = pLastCol;
 
       if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) {
+        taosMemoryFreeClear(PToFree);
+        rocksdb_free(values_list[i]);
         continue;
       }
 

From a23294d5f27838bcf57504b0818afce561b021c0 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 24 May 2024 13:40:10 +0800
Subject: [PATCH 037/132] fix:[TD-30231]case failed in sometimes

---
 source/common/src/tdataformat.c     | 2 +-
 utils/test/c/write_raw_block_test.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index ae3fe6a2a0..8d3db72fb5 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -2907,7 +2907,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
         }
       } else {
         if (varDataTLen(data + offset) > bytes) {
-          uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset),
+          uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset),
                  bytes);
           code = TSDB_CODE_PAR_VALUE_TOO_LONG;
           goto _exit;
diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c
index 162ecd229c..0f123fb560 100644
--- a/utils/test/c/write_raw_block_test.c
+++ b/utils/test/c/write_raw_block_test.c
@@ -78,7 +78,7 @@ int buildStable(TAOS* pConn) {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "insert into ntba values(now,'hello')");
+  pRes = taos_query(pConn, "insert into ntba values(now + 1s,'hello')");
   if (taos_errno(pRes) != 0) {
     printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes));
     return -1;

From a25b70c45c12a35781179c6d0e7a7abc79817198 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Mon, 27 May 2024 14:53:30 +0800
Subject: [PATCH 038/132] fix: add timeline test cases

---
 source/libs/parser/src/parTranslater.c | 51 ++++++++++++++++----------
 tests/parallel_test/cases.task         |  1 +
 tests/script/tsim/query/timeline.sim   | 51 ++++++++++++++++++++++++++
 3 files changed, 83 insertions(+), 20 deletions(-)
 create mode 100644 tests/script/tsim/query/timeline.sim

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index b326224793..c9a05a348c 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1001,14 +1001,22 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) {
   return false;
 }
 
-SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList) {
+SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) {
   SNodeList* pPartitionList = NULL;
   SNode* pNode = NULL;
-  FOREACH(pNode, pOrderList) {
-    if (pNode == pOrderList->pTail->pNode) {
-      break;
-    }
-    SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode;
+  if (pOrderList->length <= nodesNum) {
+    return NULL;
+  }
+
+  pNode = nodesListGetNode(pOrderList, nodesNum);
+  SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode;
+  if (!isPrimaryKeyImpl(pOrder->pExpr)) {
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < nodesNum; ++i) {
+    pNode = nodesListGetNode(pOrderList, i);
+    pOrder = (SOrderByExprNode*)pNode;
     nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr));
   }
 
@@ -1028,7 +1036,7 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
         return true;
       }
       if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
-        SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList);
+        SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
         bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
         nodesDestroyList(pPartitionList);
 
@@ -1041,7 +1049,7 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
   if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
     SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
     if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
-      SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList);
+      SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
       bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
       nodesDestroyList(pPartitionList);
 
@@ -6066,12 +6074,13 @@ static void resetResultTimeline(SSelectStmt* pSelect) {
     pSelect->timeLineResMode = TIME_LINE_GLOBAL;
     return;
   } else if (pSelect->pOrderByList->length > 1) {
-    pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, pSelect->pOrderByList->length - 1))->pExpr;
-    if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) ||
-        (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) {
-      pSelect->timeLineResMode = TIME_LINE_MULTI;
-      pSelect->timeLineFromOrderBy = true;
-      return;
+    for (int32_t i = 1; i < pSelect->pOrderByList->length; ++i) {
+      pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, i))->pExpr;
+      if (isPrimaryKeyImpl(pOrder)) {
+        pSelect->timeLineResMode = TIME_LINE_MULTI;
+        pSelect->timeLineFromOrderBy = true;
+        return;
+      }
     }
   }
   
@@ -6272,12 +6281,14 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
       pSetOperator->timeLineResMode = TIME_LINE_GLOBAL;
       return code;
     } else if (pSetOperator->pOrderByList->length > 1) {
-      pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, pSetOperator->pOrderByList->length - 1))->pExpr;
-      if (isPrimaryKeyImpl(pOrder)) {
-        pSetOperator->timeLineResMode = TIME_LINE_MULTI;
-        pSetOperator->timeLineFromOrderBy = true;
-        return code;
-      }
+      for (int32_t i = 1; i < pSetOperator->pOrderByList->length; ++i) {
+        pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, i))->pExpr;
+        if (isPrimaryKeyImpl(pOrder)) {
+          pSetOperator->timeLineResMode = TIME_LINE_MULTI;
+          pSetOperator->timeLineFromOrderBy = true;
+          return code;
+        }
+      
}
     }
     
     pSetOperator->timeLineResMode = TIME_LINE_NONE;
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 342389562a..866d7a152a 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -994,6 +994,7 @@
 ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py
 
 #tsim test
+,,y,script,./test.sh -f tsim/query/timeline.sim
 ,,y,script,./test.sh -f tsim/join/join.sim
 ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim
 ,,y,script,./test.sh -f tsim/parser/where.sim
diff --git a/tests/script/tsim/query/timeline.sim b/tests/script/tsim/query/timeline.sim
new file mode 100644
index 0000000000..743a6f1a9f
--- /dev/null
+++ b/tests/script/tsim/query/timeline.sim
@@ -0,0 +1,51 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create database test;
+sql use test;
+
+sql CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256));
+sql CREATE TABLE demo_1 USING demo (deviceid) TAGS ('1');
+sql CREATE TABLE demo_2 USING demo (deviceid) TAGS ('2');
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 1.0);
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 01:00:00.001', 2.0);
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 02:00:00.002', 3.0);
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-05 03:00:00.003', 4.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 5.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 01:00:00.001', 6.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 02:00:00.002', 7.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-29 03:00:00.003', 8.0);
+
+sql_error select diff(faev) from ((select ts, faev from demo union all select ts, faev from demo));
+sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts);
+sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts) partition by faev;
+sql select diff(faev) from (select _ts, faev from demo union all select _ts + 1s, faev from demo order by faev, _ts) partition by faev;
+sql_error select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by deviceid, _ts) partition by faev;
+sql select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev;
+
+sql_error select diff(faev) from (select _ts, faev from demo);
+sql_error select diff(faev) from (select _ts, faev from demo order by faev, _ts);
+sql select diff(faev) from (select _ts, faev from demo order by faev, _ts) partition by faev;
+sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts) partition by deviceid;
+sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by deviceid, _ts) partition by faev;
+sql select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY deviceid, ts) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid;
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY deviceid, ts) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

From fd34087b8cb42935f5cf1974a9f5f1c5acb84c18 Mon Sep 17 00:00:00 2001
From: wangjiaming0909 <wangjiaming0909@sina.com>
Date: Mon, 27 May 2024 16:20:13 +0800
Subject: [PATCH 039/132] fix select from union all caused crash

---
 include/libs/nodes/plannodes.h             |  1 +
 source/libs/planner/src/planLogicCreater.c |  1 +
 source/libs/planner/src/planOptimizer.c    | 30 +++++++++++++++++-----
 tests/system-test/2-query/union1.py        | 13 ++++++++++
 4 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index 18bc24d612..a691433ee6 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -183,6 +183,7 @@ typedef struct SProjectLogicNode {
   char       stmtName[TSDB_TABLE_NAME_LEN];
   bool       ignoreGroupId;
   bool       inputIgnoreGroup;
+  bool       isSetOpProj;
 } SProjectLogicNode;
 
 typedef struct SIndefRowsFuncLogicNode {
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index 23b8baf031..acc29997a2 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -1587,6 +1587,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator
     TSWAP(pProject->node.pLimit, pSetOperator->pLimit);
   }
   pProject->ignoreGroupId = true;
+  pProject->isSetOpProj = true;
 
   int32_t code = TSDB_CODE_SUCCESS;
 
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index a2a0343316..7d10c02529 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -3284,18 +3284,34 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
   SNodeList*  pNewChildTargets = nodesMakeList();
 
   if (NULL == pProjectNode->node.pParent) {
-    SNode* pProjection = NULL;
-    FOREACH(pProjection, pProjectNode->pProjections) {
-      SNode* pChildTarget = NULL;
-      FOREACH(pChildTarget, pChild->pTargets) {
-        if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
-          nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
+    SNode *pProjection = NULL, *pChildTarget = NULL;
+    bool needOrderMatch = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj;
+    bool orderMatch = true;
+    if (needOrderMatch) {
+      // For sql: select ... from (select ... union all select ...);
+      // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and
+      // union all project targets have same columns in the same order. See detail in TD-30188
+      FORBOTH(pProjection, pProjectNode->pProjections, pChildTarget, pChild->pTargets) {
+        if (!pProjection) break;
+        if (0 != strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
+          orderMatch = false;
           break;
         }
+        nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
+      }
+    } else {
+      FOREACH(pProjection, pProjectNode->pProjections) {
+        FOREACH(pChildTarget, pChild->pTargets) {
+          if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
+            nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
+            break;
+          }
+        }
       }
     }
 
-    if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets)) {
+    if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) &&
+        (!needOrderMatch || (needOrderMatch && orderMatch))) {
       nodesDestroyList(pChild->pTargets);
       pChild->pTargets = pNewChildTargets;
     } else {
diff --git a/tests/system-test/2-query/union1.py b/tests/system-test/2-query/union1.py
index 8db5ce01f3..853dfe9582 100644
--- a/tests/system-test/2-query/union1.py
+++ b/tests/system-test/2-query/union1.py
@@ -240,9 +240,22 @@ class TDTestCase:
         tdSql.error( " '' union all select c1 from ct1 " )
         # tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
 
+    def test_select_from_union_all(self):
+        tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all select ts, c8, c1 from stb1 limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select ts, c1 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select ts, c1, c8 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select ts, c8, c1, 123 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+
     def all_test(self):
         self.__test_error()
         self.union_check()
+        self.test_select_from_union_all()
 
 
     def __create_tb(self):

From 86eb1313bad3c62bdaf0ec1dee1ca5235ec6ec7a Mon Sep 17 00:00:00 2001
From: charles <flyingangel_2008@163.com>
Date: Mon, 27 May 2024 17:22:30 +0800
Subject: [PATCH 040/132] add test case for ts-4806 and complicated accuracy
 query section by charles

---
 .../query/accuracy/test_query_accuracy.py     | 77 +++++++++++++++++++
 tests/parallel_test/cases.task                |  1 +
 2 files changed, 78 insertions(+)
 create mode 100644 tests/army/community/query/accuracy/test_query_accuracy.py

diff --git a/tests/army/community/query/accuracy/test_query_accuracy.py b/tests/army/community/query/accuracy/test_query_accuracy.py
new file mode 100644
index 0000000000..f7b9c00a28
--- /dev/null
+++ b/tests/army/community/query/accuracy/test_query_accuracy.py
@@ -0,0 +1,77 @@
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.eos import *
+import random
+import string
+
+
+class TDTestCase(TBase):
+    """Add test case to verify the complicated query accuracy
+    """
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def prepare_data(self):
+        # database for case TS-4806
+        tdSql.execute("create database db_ts4806;")
+        tdSql.execute("use db_ts4806;")
+        # super table
+        tdSql.execute("create table st (ts timestamp, adl float, bdl float, cdl float, ady float, bdy float, cdy float) \
+            tags(pt_radio float, ct_ratio float, rated_cap float, ta_id varchar(128), id varchar(128), area_code \
+            varchar(128), zdy_flag int, elec_cust_name bigint,bureau_code bigint, fl_name varchar(32), classify_id \
+                varchar(128));")
+        # child table
+        tdSql.execute("create table ct_1 using st tags(1.2, 1.3, 3.4, '271000276', '30000001', '10001', 1, 10001, 2000001, 'beijing', '13169');")
+        tdSql.execute("create table ct_2 using st tags(2.1, 1.2, 3.3, '271000277', '30000002', '10002', 1, 10002, 2000002, 'shanghai', '13141');")
+        tdSql.execute("create table ct_3 using st tags(3.1, 4.2, 5.3, '271000278', '30000003', '10003', 0, 10003, 2000003, 'guangzhou', '13151');")
+        # insert data for ts4806
+        start_ts = 1705783972000
+        data = [
+            (1.1, 2.2, 3.3, 1.1, 2.2, 3.3),
+            (1.2, 2.3, 3.4, 1.2, 2.3, 3.4),
+            (1.3, 2.4, 3.5, 1.3, 2.4, 3.5),
+            (1.4, 2.5, 3.6, 1.4, 2.5, 3.6),
+            (1.5, 2.6, 3.7, 1.5, 2.6, 3.7),
+            (1.6, 2.7, 3.8, 1.6, 2.7, 3.8),
+            (1.7, 2.8, 3.9, 1.7, 2.8, 3.9),
+            (1.8, 2.9, 4.0, 1.8, 2.9, 4.0),
+            (1.9, 4.2, 4.1, 1.9, 3.0, 4.1),
+            (1.2, 3.1, 4.2, 2.0, 3.1, 4.2)
+        ]
+        index = [1, 2, 5, 0, 7, 3, 8, 4, 6, 9]
+        for ct in ['ct_1', 'ct_2']:
+            for i in range(10):
+                sql = f"insert into {ct} values"
+                for j in range(1000):
+                    sql += f"({start_ts + i * 1000 * 1000 + j * 1000}, {','.join([str(item) for item in data[index[i]]])}),"
+                sql += ";"
+                tdSql.execute(sql)
+
+    def test_ts4806(self):
+        tdSql.execute("use db_ts4806;")
+        tdSql.query("select _wstart, cj.id, count(*) from st cj where cj.ts >= '2024-01-21 04:52:52.000' and cj.ts <= ' 2024-01-21 07:39:31.000' \
+            and cj.zdy_flag = 1 and cj.id in ('30000001', '30000002') partition by cj.id event_window start with \
+            (CASE WHEN cj.adl >= cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl \
+            THEN cj.bdl ELSE cj.cdl END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap > 1 end with (CASE WHEN cj.adl >= \
+            cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl THEN cj.bdl ELSE cj.cdl \
+            END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap <= 1 HAVING count(*) >= 4 order by _wstart, cj.id;")
+        tdSql.checkRows(5)
+        tdSql.checkData(4, 1, '30000002')
+        tdSql.checkData(4, 2, 1001)
+
+    def run(self):
+        self.prepare_data()
+        self.test_ts4806()
+
+    def stop(self):
+        tdSql.execute("drop database db_ts4806;")
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 342389562a..e0e68c10ce 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -20,6 +20,7 @@
 ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f community/query/accuracy/test_query_accuracy.py
 ,,y,army,./pytest.sh python3 ./test.py -f community/insert/insert_basic.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py

From a9ca820b06964c683ae4e1a27c88e19f5c82f190 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 27 May 2024 22:23:01 +0800
Subject: [PATCH 041/132] fix(query): check if pSliceIno->pPrevGroupKey is
 initialized or not.

---
 source/libs/executor/src/timesliceoperator.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c
index 9a84127be6..9f421d6de3 100644
--- a/source/libs/executor/src/timesliceoperator.c
+++ b/source/libs/executor/src/timesliceoperator.c
@@ -511,7 +511,6 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB
   }
 
   pInfo->isPrevRowSet = false;
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -825,8 +824,12 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato
   SSDataBlock* pResBlock = pSliceInfo->pRes;
   SInterval*   pInterval = &pSliceInfo->interval;
 
-  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
-         pSliceInfo->fillType != TSDB_FILL_LINEAR) {
+  if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR ||
+      pSliceInfo->pPrevGroupKey == NULL) {
+    return;
+  }
+
+  while (pSliceInfo->current <= pSliceInfo->win.ekey) {
     genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false);
     pSliceInfo->current =
         taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
@@ -1068,6 +1071,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
 
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
 
+//  int32_t code = initKeeperInfo(pSliceInfo, pBlock, &pOperator->exprSupp);
+
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 

From be107b204c2ce9bcd37454caed05da919dd351d6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 28 May 2024 09:54:22 +0800
Subject: [PATCH 042/132] enh(stream): support retrieve checkpoint data
 remotely when start stream tasks in vnodes.

---
 include/common/rsync.h                        |   2 +-
 include/common/tmsgdef.h                      |   1 +
 include/dnode/vnode/tqCommon.h                |   1 +
 include/libs/stream/streammsg.h               |  18 +-
 include/libs/stream/tstream.h                 |  32 +-
 source/common/src/rsync.c                     |   8 +-
 source/dnode/mgmt/mgmt_snode/src/smHandle.c   |   2 +
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c   |   2 +
 source/dnode/vnode/src/inc/vnodeInt.h         |   1 +
 source/dnode/vnode/src/sma/smaRollup.c        |   3 +-
 source/dnode/vnode/src/tq/tq.c                |  22 +-
 source/dnode/vnode/src/tqCommon/tqCommon.c    |  63 +++-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c    |   2 +-
 source/dnode/vnode/src/vnd/vnodeSvr.c         |   4 +
 source/libs/stream/inc/streamInt.h            |  35 ++
 source/libs/stream/src/streamBackendRocksdb.c |  14 +-
 source/libs/stream/src/streamCheckStatus.c    |   2 +-
 source/libs/stream/src/streamCheckpoint.c     | 341 +++++++++++++++---
 source/libs/stream/src/streamDispatch.c       | 182 +++++++---
 source/libs/stream/src/streamExec.c           |   2 +-
 source/libs/stream/src/streamMeta.c           |  10 +-
 source/libs/stream/src/streamTask.c           | 123 ++++++-
 source/libs/stream/src/streammsg.c            |   8 +-
 23 files changed, 716 insertions(+), 162 deletions(-)

diff --git a/include/common/rsync.h b/include/common/rsync.h
index f613a35f48..0840b51793 100644
--- a/include/common/rsync.h
+++ b/include/common/rsync.h
@@ -13,7 +13,7 @@ extern "C" {
 
 void    stopRsync();
 void    startRsync();
-int32_t uploadRsync(const char* id, const char* path);
+int32_t uploadByRsync(const char* id, const char* path);
 int32_t downloadRsync(const char* id, const char* path);
 int32_t deleteRsync(const char* id);
 
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index 2d07b56e4c..d1ac2c79c3 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -321,6 +321,7 @@
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE_TRIGGER, "stream-retri-trigger", NULL, NULL)
 
   TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
 
diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h
index 0cde499a6b..6d52b10182 100644
--- a/include/dnode/vnode/tqCommon.h
+++ b/include/dnode/vnode/tqCommon.h
@@ -36,6 +36,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
 int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta);
 int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta);
 int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
 int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
 int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
diff --git a/include/libs/stream/streammsg.h b/include/libs/stream/streammsg.h
index 5436442284..87c756b10c 100644
--- a/include/libs/stream/streammsg.h
+++ b/include/libs/stream/streammsg.h
@@ -22,17 +22,17 @@
 extern "C" {
 #endif
 
-typedef struct SStreamChildEpInfo {
+typedef struct SStreamUpstreamEpInfo {
   int32_t nodeId;
   int32_t childId;
   int32_t taskId;
   SEpSet  epSet;
   bool    dataAllowed;  // denote if the data from this upstream task is allowed to put into inputQ, not serialize it
   int64_t stage;  // upstream task stage value, to denote if the upstream node has restart/replica changed/transfer
-} SStreamChildEpInfo;
+} SStreamUpstreamEpInfo;
 
-int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
-int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
+int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo);
+int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamUpstreamEpInfo* pInfo);
 
 // mndTrigger: denote if this checkpoint is triggered by mnode or as requested from tasks when transfer-state finished
 typedef struct {
@@ -171,6 +171,16 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp);
 int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pRsp);
 void    tCleanupStreamHbMsg(SStreamHbMsg* pMsg);
 
+typedef struct SRetrieveChkptTriggerReq {
+  SMsgHead head;
+  int64_t  streamId;
+  int64_t  checkpointId;
+  int32_t  upstreamNodeId;
+  int32_t  upstreamTaskId;
+  int32_t  downstreamNodeId;
+  int64_t  downstreamTaskId;
+} SRetrieveChkptTriggerReq;
+
 typedef struct {
   SMsgHead head;
   int64_t  streamId;
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index ea0f1824b3..185ab7ad51 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -61,10 +61,11 @@ extern "C" {
 // the load and start stream task should be executed after snode has started successfully, since the load of stream
 // tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources.
 
-typedef struct SStreamTask      SStreamTask;
-typedef struct SStreamQueue     SStreamQueue;
-typedef struct SStreamTaskSM    SStreamTaskSM;
-typedef struct SStreamQueueItem SStreamQueueItem;
+typedef struct SStreamTask           SStreamTask;
+typedef struct SStreamQueue          SStreamQueue;
+typedef struct SStreamTaskSM         SStreamTaskSM;
+typedef struct SStreamQueueItem      SStreamQueueItem;
+typedef struct SActiveCheckpointInfo SActiveCheckpointInfo;
 
 #define SSTREAM_TASK_VER                  4
 #define SSTREAM_TASK_INCOMPATIBLE_VER     1
@@ -270,13 +271,10 @@ typedef struct SCheckpointInfo {
   int64_t checkpointTime;  // latest checkpoint time
   int64_t processedVer;
   int64_t nextProcessVer;  // current offset in WAL, not serialize it
-  int64_t failedId;        // record the latest failed checkpoint id
-  int64_t checkpointingId;
-  int32_t downstreamAlignNum;
   int32_t numOfNotReady;
-  bool    dispatchCheckpointTrigger;
+
+  SActiveCheckpointInfo* pActiveInfo;
   int64_t msgVer;
-  int32_t transId;
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
@@ -436,7 +434,6 @@ struct SStreamTask {
   SHistoryTaskInfo    hTaskInfo;
   STaskId             streamTaskId;
   STaskExecStatisInfo execInfo;
-  SArray*             pReadyMsgList;  // SArray<SStreamChkptReadyInfo*>
   TdThreadMutex       lock;           // secure the operation of set task status and puting data into inputQ
   SMsgCb*             pMsgCb;         // msg handle
   SStreamState*       pState;         // state backend
@@ -619,7 +616,8 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
 int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
 int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
 
-SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);
+SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);
+SEpSet*                streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
 
 void    streamTaskInputFail(SStreamTask* pTask);
 
@@ -672,6 +670,17 @@ int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
 int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration);
 bool    streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
 
+// checkpoint related
+int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId);
+int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId);
+int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId);
+bool    streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId);
+void    streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal);
+void    streamTaskInitTriggerDispatchInfo(SStreamTask* pTask);
+void    streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId);
+int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t checkpointType, int32_t dstTaskId, int32_t vgId,
+                                           SEpSet* pEpset);
+
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
 
 // common
@@ -682,6 +691,7 @@ int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstre
 void    streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask);
 int32_t streamTaskReleaseState(SStreamTask* pTask);
 int32_t streamTaskReloadState(SStreamTask* pTask);
+void    streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId);
 void    streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
 void    streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
 int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key);
diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index c4d14a6c2c..1b03b142e0 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -153,7 +153,7 @@ void startRsync() {
   uDebug("[rsync] start server successful");
 }
 
-int32_t uploadRsync(const char* id, const char* path) {
+int32_t uploadByRsync(const char* id, const char* path) {
   int64_t st = taosGetTimestampMs();
   char    command[PATH_MAX] = {0};
 
@@ -196,6 +196,7 @@ int32_t uploadRsync(const char* id, const char* path) {
   return code;
 }
 
+// abort from retry if quit
 int32_t downloadRsync(const char* id, const char* path) {
   int64_t st = taosGetTimestampMs();
   int32_t MAX_RETRY = 60;
@@ -220,11 +221,10 @@ int32_t downloadRsync(const char* id, const char* path) {
   uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command);
 
   while(times++ < MAX_RETRY) {
-
     code = execCommand(command);
     if (code != TSDB_CODE_SUCCESS) {
-      uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, code:%d," ERRNO_ERR_FORMAT, id, path, code,
-             ERRNO_ERR_DATA);
+      uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id,
+             path, times, code, ERRNO_ERR_DATA);
       taosSsleep(1);
     } else {
       int32_t el = taosGetTimestampMs() - st;
diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
index b3c8ef4017..9b07b6a3d8 100644
--- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c
+++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
@@ -89,6 +89,8 @@ SArray *smGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index d9175bf5fe..bfc9e92293 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -961,6 +961,8 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 9439f7f179..f3bdc98994 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -257,6 +257,7 @@ int     tqScanWalAsync(STQ* pTq, bool ckPause);
 int32_t tqStopStreamTasksAsync(STQ* pTq);
 int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp);
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskRetrieveTriggerMsg(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg);
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 5441d0c4c1..b8d0e30d30 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -1285,7 +1285,8 @@ _checkpoint:
         if (pItem && pItem->pStreamTask) {
           SStreamTask *pTask = pItem->pStreamTask;
           // atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1);
-          pTask->chkInfo.checkpointingId = checkpointId;
+          streamTaskSetActiveCheckpointInfo(pTask, checkpointId);
+
           pTask->chkInfo.checkpointId = checkpointId;  // 1pTask->checkpointingId;
           pTask->chkInfo.checkpointVer = pItem->submitReqVer;
           pTask->info.triggerParam = pItem->fetchResultVer;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index a59a235c50..037c2a7b7a 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -725,10 +725,6 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   }
 
   pTask->pBackend = NULL;
-//  code = tqExpandStreamTask(pTask, pTq->pStreamMeta);
-//  if (code != TSDB_CODE_SUCCESS) {
-//    return code;
-//  }
 
   // sink
   STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
@@ -1092,6 +1088,10 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   int32_t      len = pMsg->contLen - sizeof(SMsgHead);
   int32_t      code = 0;
 
+//  if (pTq->pStreamMeta->vgId == 2) {
+//    ASSERT(0);
+//  }
+
   // disable auto rsp to mnode
   pRsp->info.handle = NULL;
 
@@ -1140,10 +1140,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   }
 
   if (pTask->status.downstreamReady != 1) {
-    pTask->chkInfo.failedId = req.checkpointId;  // record the latest failed checkpoint id
-    pTask->chkInfo.checkpointingId = req.checkpointId;
-    pTask->chkInfo.transId = req.transId;
-
+    streamTaskSetFailedChkptInfo(pTask, req.transId, req.checkpointId);  // record the latest failed checkpoint id
     tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpointId:%" PRId64
             ", transId:%d set it failed",
             pTask->id.idStr, req.checkpointId, req.transId);
@@ -1182,9 +1179,12 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 
   // check if the checkpoint msg already sent or not.
   if (status == TASK_STATUS__CK) {
+    int64_t checkpointId = 0;
+    streamTaskGetActiveCheckpointInfo(pTask, NULL, &checkpointId);
+
     tqWarn("s-task:%s repeatly recv checkpoint-source msg checkpointId:%" PRId64
            " transId:%d already handled, ignore msg and continue process checkpoint",
-           pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId);
+           pTask->id.idStr, checkpointId, req.transId);
 
     taosThreadMutexUnlock(&pTask->lock);
     streamMetaReleaseTask(pMeta, pTask);
@@ -1244,6 +1244,10 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
   return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg);
 }
 
+int32_t tqProcessTaskRetrieveTriggerMsg(STQ* pTq, SRpcMsg* pMsg) {
+  return tqStreamTaskProcessRetrieveTriggerReq(pTq->pStreamMeta, pMsg);
+}
+
 // this function is needed, do not try to remove it.
 int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) {
   return tqStreamProcessStreamHbRsp(pTq->pStreamMeta, pMsg);
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index b5b5ef8755..b9c0589dc5 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -699,9 +699,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
   }
 
   if (isLeader && !tsDisableStream) {
-    streamMetaResetTaskStatus(pMeta);
     streamMetaWUnLock(pMeta);
-
     streamMetaStartAllTasks(pMeta, tqExpandStreamTask);
   } else {
     streamMetaResetStartInfo(&pMeta->startInfo);
@@ -839,13 +837,19 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
   // clear flag set during do checkpoint, and open inputQ for all upstream tasks
   SStreamTaskState *pState = streamTaskGetStatus(pTask);
   if (pState->state == TASK_STATUS__CK) {
+    int32_t tranId = 0;
+    int64_t activeChkId = 0;
+    streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId);
+
     tqDebug("s-task:%s reset task status from checkpoint, current checkpointingId:%" PRId64 ", transId:%d",
-            pTask->id.idStr, pTask->chkInfo.checkpointingId, pTask->chkInfo.transId);
+            pTask->id.idStr, activeChkId, tranId);
+
     streamTaskSetStatusReady(pTask);
   } else if (pState->state == TASK_STATUS__UNINIT) {
     tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
     ASSERT(pTask->status.downstreamReady == 0);
-    /*int32_t ret = */ streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT);
+//    /*int32_t ret = */ streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT);
+    tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, pTask->id.streamId, pTask->id.taskId);
   } else {
     tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name);
   }
@@ -856,6 +860,57 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
+  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg;
+
+  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId);
+  if (pTask == NULL) {
+    tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
+            " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
+            pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  tqDebug("s-task:0x%x recv retrieve checkpoint-trigger msg from downstream s-task:0x%x, checkpointId:%" PRId64,
+          pReq->upstreamTaskId, (int32_t)pReq->downstreamTaskId, pReq->checkpointId);
+
+  SStreamTaskState* pState = streamTaskGetStatus(pTask);
+  if (pState->state == TASK_STATUS__CK) {  // recv the checkpoint-source/trigger already
+    int32_t transId = 0;
+    int64_t checkpointId = 0;
+
+    streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId);
+    ASSERT (checkpointId == pReq->checkpointId);
+
+    if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) {
+      // re-send the lost checkpoint-trigger msg to downstream task
+      SEpSet* pEpset = streamTaskGetDownstreamEpInfo(pTask, pReq->downstreamTaskId);
+      streamTaskSendCheckpointTriggerMsg(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pReq->downstreamTaskId,
+                                         pReq->downstreamNodeId, pEpset);
+    } else { // not send checkpoint-trigger yet, wait
+      int32_t recv = 0, total = 0;
+      streamTaskGetTriggerRecvStatus(pTask, &recv, &total);
+
+      if (recv == total) {  // add the ts info
+        tqWarn("s-task:%s all upstream send checkpoint-source/trigger, but not processed yet, wait", pTask->id.idStr);
+      } else {
+        tqWarn(
+            "s-task:%s not all upstream send checkpoint-source/trigger, total recv:%d/%d, wait for all upstream "
+            "sending checkpoint-source/trigger",
+            pTask->id.idStr, recv, total);
+      }
+    }
+  } else {  // upstream not recv the checkpoint-source/trigger till now
+    ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT);
+    tqWarn(
+        "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
+        "upstream sending checkpoint-source/trigger",
+        pTask->id.idStr);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
   SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index ea742108aa..fd1bb391b2 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -422,7 +422,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
   }
 
   // STREAM ============
-  vInfo("vgId:%d stream task start", vgId);
+  vInfo("vgId:%d stream task start to take snapshot", vgId);
   if (!pReader->streamTaskDone) {
     if (pReader->pStreamTaskReader == NULL) {
       code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader);
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index f6b32c5543..02343206ad 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -842,12 +842,16 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
       return tqProcessTaskScanHistory(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_CHECKPOINT_READY:
       return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
+    case TDMT_STREAM_RETRIEVE_TRIGGER:
+      return tqProcessTaskRetrieveTriggerMsg(pVnode->pTq, pMsg);
     case TDMT_MND_STREAM_HEARTBEAT_RSP:
       return tqProcessStreamHbRsp(pVnode->pTq, pMsg);
     case TDMT_MND_STREAM_REQ_CHKPT_RSP:
       return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
       return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
+    case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:
+      return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
     case TDMT_VND_GET_STREAM_PROGRESS:
       return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
     default:
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 10bdccdb29..68c3ab1a6b 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -52,6 +52,18 @@ extern "C" {
 #define stTrace(...) do { if (stDebugFlag & DEBUG_TRACE) { taosPrintLog("STM ", DEBUG_TRACE, stDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
+struct SActiveCheckpointInfo {
+  TdThreadMutex lock;
+  int32_t       transId;
+  int64_t       firstRecvTs;  // first time to recv checkpoint trigger info
+  int64_t       activeId;     // current active checkpoint id
+  int64_t       failedId;
+  bool          dispatchTrigger;
+  SArray*       pDispatchTriggerList;   // SArray<STaskTriggerSendInfo>
+  SArray*       pReadyMsgList;   // SArray<SStreamChkptReadyInfo*>
+  tmr_h        pCheckTmr;
+};
+
 typedef struct {
   int8_t       type;
   SSDataBlock* pBlock;
@@ -81,6 +93,24 @@ struct STokenBucket {
   int64_t quotaFillTimestamp;  // fill timestamp
 };
 
+typedef struct {
+  int32_t upStreamTaskId;
+  SEpSet  upstreamNodeEpset;
+  int32_t nodeId;
+  SRpcMsg msg;
+  int64_t recvTs;
+  int32_t transId;
+  int64_t checkpointId;
+} SStreamChkptReadyInfo;
+
+typedef struct {
+  int64_t sendTs;
+  int64_t recvTs;
+  bool    recved;
+  int32_t nodeId;
+  int32_t taskId;
+} STaskTriggerSendInfo;
+
 struct SStreamQueue {
   STaosQueue* pQueue;
   STaosQall*  qall;
@@ -113,6 +143,9 @@ void    destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups);
 int32_t getNumOfDispatchBranch(SStreamTask* pTask);
 void    clearBufferedDispatchMsg(SStreamTask* pTask);
 
+int32_t streamTaskBuildAndSendTriggerMsg(SStreamTask* pTask, const SStreamDataBlock* pData, int32_t dstTaskId,
+                                         int32_t vgId, SEpSet* pEpset);
+
 int32_t           streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
 SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg);
 SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize,
@@ -131,11 +164,13 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask);
 
 void    streamTaskSetFailedCheckpointId(SStreamTask* pTask);
 int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask);
+int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask);
 int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*);
 STaskId streamTaskGetTaskId(const SStreamTask* pTask);
 void    streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo);
 void    streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo);
 int32_t streamTaskResetTimewindowFilter(SStreamTask* pTask);
+void    streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo);
 
 void              streamClearChkptReadyMsg(SStreamTask* pTask);
 EExtractDataCode  streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks,
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 28db0a76c6..87fb615d5f 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -386,12 +386,12 @@ int32_t rebuildFromRemoteChkp_s3(const char* key, char* chkpPath, int64_t chkpId
   return code;
 }
 
-int32_t rebuildFromRemoteCheckpoint(const char* key, char* chkpPath, int64_t checkpointId, char* defaultPath) {
+int32_t rebuildFromRemoteCheckpoint(const char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) {
   ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
   if (type == DATA_UPLOAD_S3) {
-    return rebuildFromRemoteChkp_s3(key, chkpPath, checkpointId, defaultPath);
+    return rebuildFromRemoteChkp_s3(key, chkptPath, checkpointId, defaultPath);
   } else if (type == DATA_UPLOAD_RSYNC) {
-    return rebuildFromRemoteChkp_rsync(key, chkpPath, checkpointId, defaultPath);
+    return rebuildFromRemoteChkp_rsync(key, chkptPath, checkpointId, defaultPath);
   } else {
     stError("%s no remote backup checkpoint data for:%" PRId64, key, checkpointId);
   }
@@ -538,7 +538,9 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
     taosMulMkDir(defaultPath);
   }
 
-  char* checkpointRoot = taosMemoryCalloc(1, strlen(path) + 256);
+  int32_t pathLen = strlen(path) + 256;
+
+  char* checkpointRoot = taosMemoryCalloc(1, pathLen);
   sprintf(checkpointRoot, "%s%s%s", prefixPath, TD_DIRSEP, "checkpoints");
 
   if (!taosIsDir(checkpointRoot)) {
@@ -548,9 +550,9 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
 
   stDebug("%s check local backend dir:%s, checkpointId:%" PRId64 " succ", key, defaultPath, chkptId);
 
-  char* chkptPath = taosMemoryCalloc(1, strlen(path) + 256);
+  char* chkptPath = taosMemoryCalloc(1, pathLen);
   if (chkptId > 0) {
-    sprintf(chkptPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId);
+    snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId);
 
     code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath);
     if (code != 0) {
diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c
index 4a8ca69ba5..f083ff8a61 100644
--- a/source/libs/stream/src/streamCheckStatus.c
+++ b/source/libs/stream/src/streamCheckStatus.c
@@ -40,7 +40,7 @@ static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t
 
 int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
                               int64_t* oldStage) {
-  SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
+  SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
   ASSERT(pInfo != NULL);
 
   *oldStage = pInfo->stage;
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index d09e5bf477..43b39b8574 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -20,9 +20,9 @@
 
 typedef struct {
   ECHECKPOINT_BACKUP_TYPE type;
-  char*                   taskId;
-  int64_t                 chkpId;
 
+  char*        taskId;
+  int64_t      chkpId;
   SStreamTask* pTask;
   int64_t      dbRefId;
   void*        pMeta;
@@ -35,22 +35,33 @@ static int32_t deleteCheckpoint(const char* id);
 static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName);
 static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask);
 static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType);
-static int32_t streamAlignCheckpoint(SStreamTask* pTask);
+static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList);
+static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType);
+static void checkpointTriggerMonitorFn(void* param, void* tmrId);
 
-int32_t streamAlignCheckpoint(SStreamTask* pTask) {
-  int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
-  int64_t old = atomic_val_compare_exchange_32(&pTask->chkInfo.downstreamAlignNum, 0, num);
-  if (old == 0) {
-    stDebug("s-task:%s set initial align upstream num:%d", pTask->id.idStr, num);
+bool streamTaskIsAllUpstreamSendTrigger(SStreamTask* pTask) {
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+  int32_t numOfUpstreams = taosArrayGetSize(pTask->upstreamInfo.pList);
+  bool allSend = true;
+
+  taosThreadMutexLock(&pActiveInfo->lock);
+  int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList);
+
+  if (numOfRecv < numOfUpstreams) {
+    stDebug("s-task:%s received checkpoint-trigger block, idx:%d, %d upstream tasks not send yet, total:%d",
+            pTask->id.idStr, pTask->info.selfChildId, (numOfUpstreams - numOfRecv), numOfUpstreams);
+    allSend = false;
   }
 
-  return atomic_sub_fetch_32(&pTask->chkInfo.downstreamAlignNum, 1);
+  taosThreadMutexUnlock(&pActiveInfo->lock);
+  return allSend;
 }
 
-int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) {
+SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType) {
   SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock));
   if (pChkpoint == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
   }
 
   pChkpoint->type = checkpointType;
@@ -58,12 +69,13 @@ int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) {
   SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
   if (pBlock == NULL) {
     taosFreeQitem(pChkpoint);
-    return TSDB_CODE_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
   }
 
   pBlock->info.type = STREAM_CHECKPOINT;
-  pBlock->info.version = pTask->chkInfo.checkpointingId;
-  pBlock->info.window.ekey = pBlock->info.window.skey = pTask->chkInfo.transId;  // NOTE: set the transId
+  pBlock->info.version = pTask->chkInfo.pActiveInfo->activeId;
+  pBlock->info.window.ekey = pBlock->info.window.skey = pTask->chkInfo.pActiveInfo->transId;  // NOTE: set the transId
   pBlock->info.rows = 1;
   pBlock->info.childId = pTask->info.selfChildId;
 
@@ -71,6 +83,14 @@ int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) {
   taosArrayPush(pChkpoint->blocks, pBlock);
 
   taosMemoryFree(pBlock);
+  terrno = 0;
+
+  return pChkpoint;
+}
+
+int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) {
+  SStreamDataBlock* pChkpoint = createChkptTriggerBlock(pTask, checkpointType);
+
   if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pChkpoint) < 0) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -87,23 +107,43 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo
   int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
   ASSERT(code == TSDB_CODE_SUCCESS);
 
-  pTask->chkInfo.transId = pReq->transId;
-  pTask->chkInfo.checkpointingId = pReq->checkpointId;
+  pTask->chkInfo.pActiveInfo->transId = pReq->transId;
+  pTask->chkInfo.pActiveInfo->activeId = pReq->checkpointId;
   pTask->chkInfo.numOfNotReady = streamTaskGetNumOfDownstream(pTask);
   pTask->chkInfo.startTs = taosGetTimestampMs();
   pTask->execInfo.checkpoint += 1;
 
   // 2. Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task
+  // and this is the last item in the inputQ.
   return appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER);
 }
 
+int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t checkpointType, int32_t dstTaskId, int32_t vgId,
+                                           SEpSet* pEpset) {
+  SStreamDataBlock* pChkpoint = createChkptTriggerBlock(pTask, checkpointType);
+
+  pChkpoint->srcTaskId = pTask->id.taskId;
+  pChkpoint->srcVgId = pTask->pMeta->vgId;
+
+  int32_t code = streamTaskBuildAndSendTriggerMsg(pTask, pChkpoint, dstTaskId, vgId, pEpset);
+  if (code == TSDB_CODE_SUCCESS) {
+    stDebug("s-task:%s build and send checkpoint-trigger dispatch msg succ, stage:%" PRId64, pTask->id.idStr,
+            pTask->pMeta->stage);
+  } else {
+    // todo handle send data failure
+    stError("s-task:%s failed to build and send trigger msg", pTask->id.idStr);
+  }
+
+  return code;
+}
+
 int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) {
   pBlock->srcTaskId = pTask->id.taskId;
   pBlock->srcVgId = pTask->pMeta->vgId;
 
   int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
   if (code == 0) {
-    ASSERT(pTask->chkInfo.dispatchCheckpointTrigger == false);
+    ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false);
     streamDispatchStreamBlock(pTask);
   } else {
     stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code));
@@ -127,8 +167,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
 
   // set task status
   if (streamTaskGetStatus(pTask)->state != TASK_STATUS__CK) {
-    pTask->chkInfo.checkpointingId = checkpointId;
-    pTask->chkInfo.transId = transId;
+    pTask->chkInfo.pActiveInfo->activeId = checkpointId;
+    pTask->chkInfo.pActiveInfo->transId = transId;
 
     code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
     if (code != TSDB_CODE_SUCCESS) {
@@ -136,12 +176,20 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       streamFreeQitem((SStreamQueueItem*)pBlock);
       return code;
     }
+
+    SActiveCheckpointInfo* pActive = pTask->chkInfo.pActiveInfo;
+    if (pActive->pCheckTmr == NULL) {
+      pActive->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 10000, pTask, streamTimer);
+    } else {
+      taosTmrReset(checkpointTriggerMonitorFn, 10000, pTask, streamTimer, &pActive->pCheckTmr);
+    }
   }
 
   // todo fix race condition: set the status and append checkpoint block
   int32_t taskLevel = pTask->info.taskLevel;
   if (taskLevel == TASK_LEVEL__SOURCE) {
     int8_t type = pTask->outputInfo.type;
+
     if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
       stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
       continueDispatchCheckpointTriggerBlock(pBlock, pTask);
@@ -161,23 +209,19 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId);
 
     // there are still some upstream tasks not send checkpoint request, do nothing and wait for then
-    int32_t notReady = streamAlignCheckpoint(pTask);
-    int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
-    if (notReady > 0) {
-      stDebug("s-task:%s received checkpoint block, idx:%d, %d upstream tasks not send checkpoint info yet, total:%d",
-              id, pTask->info.selfChildId, notReady, num);
+    bool allSend = streamTaskIsAllUpstreamSendTrigger(pTask);
+    if (!allSend) {
       streamFreeQitem((SStreamQueueItem*)pBlock);
       return code;
     }
 
+    int32_t num = streamTaskGetNumOfUpstream(pTask);
     if (taskLevel == TASK_LEVEL__SINK) {
-      stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, send ready msg to upstream",
-              id, num);
+      stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num);
       streamFreeQitem((SStreamQueueItem*)pBlock);
       streamTaskBuildCheckpoint(pTask);
     } else {  // source & agg tasks need to forward the checkpoint msg downwards
-      stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, continue forwards msg", id,
-              num);
+      stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num);
 
       // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task
       // can start local checkpoint procedure
@@ -216,14 +260,10 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) {
 }
 
 void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
-  pTask->chkInfo.checkpointingId = 0;  // clear the checkpoint id
-  pTask->chkInfo.failedId = 0;
   pTask->chkInfo.startTs = 0;  // clear the recorded start time
   pTask->chkInfo.numOfNotReady = 0;
-  pTask->chkInfo.transId = 0;
-  pTask->chkInfo.dispatchCheckpointTrigger = false;
-  pTask->chkInfo.downstreamAlignNum = 0;
 
+  streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo);
   streamTaskOpenAllUpstreamInput(pTask);  // open inputQ for all upstream tasks
   if (clearChkpReadyMsg) {
     streamClearChkptReadyMsg(pTask);
@@ -317,9 +357,9 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin
 }
 
 void streamTaskSetFailedCheckpointId(SStreamTask* pTask) {
-  pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId;
+  pTask->chkInfo.pActiveInfo->failedId = pTask->chkInfo.pActiveInfo->activeId;
   stDebug("s-task:%s mark the checkpointId:%" PRId64 " (transId:%d) failed", pTask->id.idStr,
-          pTask->chkInfo.checkpointingId, pTask->chkInfo.transId);
+          pTask->chkInfo.pActiveInfo->activeId, pTask->chkInfo.pActiveInfo->transId);
 }
 
 static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* list) {
@@ -363,7 +403,7 @@ int32_t uploadCheckpointData(void* param) {
   SAsyncUploadArg* pParam = param;
   char*            path = NULL;
   int32_t          code = 0;
-  SArray*          toDelFiles = taosArrayInit(4, sizeof(void*));
+  SArray*          toDelFiles = taosArrayInit(4, POINTER_BYTES);
   char*            taskStr = pParam->taskId ? pParam->taskId : "NULL";
 
   void* pBackend = taskAcquireDb(pParam->dbRefId);
@@ -387,10 +427,10 @@ int32_t uploadCheckpointData(void* param) {
 
   if (code == TSDB_CODE_SUCCESS) {
     code = streamTaskUploadCheckpoint(pParam->taskId, path);
-    if (code != TSDB_CODE_SUCCESS) {
-      stError("s-task:%s failed to upload checkpoint data:%s, checkpointId:%" PRId64, taskStr, path, pParam->chkpId);
+    if (code == TSDB_CODE_SUCCESS) {
+      stDebug("s-task:%s upload checkpointId:%" PRId64 " to remote succ", taskStr, pParam->chkpId);
     } else {
-      stDebug("s-task:%s backup checkpointId:%"PRId64" to remote succ", taskStr, pParam->chkpId);
+      stError("s-task:%s failed to upload checkpointId:%" PRId64 " data:%s", taskStr, pParam->chkpId, path);
     }
   }
 
@@ -403,19 +443,25 @@ int32_t uploadCheckpointData(void* param) {
     for (int i = 0; i < size; i++) {
       char* pName = taosArrayGetP(toDelFiles, i);
       code = deleteCheckpointFile(pParam->taskId, pName);
-      stDebug("s-task:%s try to del file: %s", taskStr, pName);
       if (code != 0) {
+        stDebug("s-task:%s failed to del file: %s", taskStr, pName);
         break;
       }
     }
+
+    stDebug("s-task:%s remove redundant files done", taskStr);
   }
 
   taosArrayDestroyP(toDelFiles, taosMemoryFree);
 
-  stDebug("s-task:%s remove local checkpoint dir:%s", taskStr, path);
-  taosRemoveDir(path);
-  taosMemoryFree(path);
+  if (code == TSDB_CODE_SUCCESS) {
+    stDebug("s-task:%s remove local checkpointId:%" PRId64 " data %s", taskStr, pParam->chkpId, path);
+    taosRemoveDir(path);
+  } else {
+    stDebug("s-task:%s update checkpointId:%" PRId64 " keep local checkpoint data", taskStr, pParam->chkpId);
+  }
 
+  taosMemoryFree(path);
   taosMemoryFree(pParam->taskId);
   taosMemoryFree(pParam);
 
@@ -446,7 +492,7 @@ int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointI
 int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
   int32_t      code = TSDB_CODE_SUCCESS;
   int64_t      startTs = pTask->chkInfo.startTs;
-  int64_t      ckId = pTask->chkInfo.checkpointingId;
+  int64_t      ckId = pTask->chkInfo.pActiveInfo->activeId;
   const char*  id = pTask->id.idStr;
   bool         dropRelHTask = (streamTaskGetPrevStatus(pTask) == TASK_STATUS__HALT);
   SStreamMeta* pMeta = pTask->pMeta;
@@ -510,6 +556,211 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
   return code;
 }
 
+void checkpointTriggerMonitorFn(void* param, void* tmrId) {
+  SStreamTask* pTask = param;
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+
+  int32_t vgId = pTask->pMeta->vgId;
+  int64_t now = taosGetTimestampMs();
+  stDebug("s-task:%s vgId:%d checkpoint-trigger monit start, ts:%" PRId64, pTask->id.idStr, vgId, now);
+
+  taosThreadMutexLock(&pTask->lock);
+  SStreamTaskState* pState = streamTaskGetStatus(pTask);
+  if (pState->state == TASK_STATUS__CK) {
+    stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger", pTask->id.idStr, vgId);
+    taosThreadMutexUnlock(&pTask->lock);
+    return;
+  }
+  taosThreadMutexUnlock(&pTask->lock);
+
+  taosThreadMutexLock(&pActiveInfo->lock);
+
+  // send msg to retrieve checkpoint trigger msg
+  SArray* pList = pTask->upstreamInfo.pList;
+  ASSERT(pTask->info.taskLevel > TASK_LEVEL__SOURCE);
+  SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo));
+
+  for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pList, i);
+
+    bool recved = false;
+    for(int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) {
+      SStreamChkptReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j);
+      if (pInfo->nodeId == pReady->nodeId) {
+        recved = true;
+        break;
+      }
+    }
+
+    if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message
+      streamTaskOpenUpstreamInput(pTask, pInfo->taskId);
+      taosArrayPush(pNotSendList, pInfo);
+    }
+  }
+
+  // do send retrieve checkpoint trigger msg to upstream
+  doSendRetrieveTriggerMsg(pTask, pNotSendList);
+  taosThreadMutexUnlock(&pActiveInfo->lock);
+
+  // check every 100ms
+  if (taosArrayGetSize(pNotSendList) > 0) {
+    taosTmrReset(checkpointTriggerMonitorFn, 10000, pTask, streamTimer, &pActiveInfo->pCheckTmr);
+  }
+
+  taosArrayDestroy(pNotSendList);
+}
+
+int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
+  int32_t     code = 0;
+  int32_t     vgId = pTask->pMeta->vgId;
+  const char* pId = pTask->id.idStr;
+
+  for (int32_t i = 0; i < taosArrayGetSize(pNotSendList); i++) {
+    SStreamUpstreamEpInfo* pUpstreamTask = taosArrayGet(pNotSendList, i);
+
+    SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq));
+    if (pReq == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId);
+      continue;
+    }
+
+    pReq->head.vgId = htonl(pUpstreamTask->nodeId);
+    pReq->streamId = pTask->id.streamId;
+    pReq->downstreamTaskId = pTask->id.taskId;
+    pReq->downstreamNodeId = vgId;
+    pReq->upstreamTaskId = pUpstreamTask->taskId;
+    pReq->upstreamNodeId = pUpstreamTask->nodeId;
+    pReq->checkpointId = pTask->chkInfo.pActiveInfo->activeId;
+
+    SRpcMsg rpcMsg = {0};
+    initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE, pReq, sizeof(SRetrieveChkptTriggerReq));
+
+    code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) {
+  SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
+  if (pStatus->state != TASK_STATUS__CK) {
+    return false;
+  }
+
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+
+  taosThreadMutexLock(&pInfo->lock);
+  if (!pInfo->dispatchTrigger) {
+    taosThreadMutexUnlock(&pInfo->lock);
+    return false;
+  }
+
+  for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
+    STaskTriggerSendInfo* pSendInfo = taosArrayGet(pInfo->pDispatchTriggerList, i);
+    if (pSendInfo->nodeId != downstreamNodeId) {
+      continue;
+    }
+
+    // has send trigger msg to downstream node,
+    if (pSendInfo->recved) {
+      stWarn("s-task:%s checkpoint-trigger msg send at:%"PRId64" and recv confirmed, checkpointId:%"PRId64 ", transId:%d",
+          pTask->id.idStr, pSendInfo->sendTs, pInfo->activeId, pInfo->transId);
+    } else {
+      stWarn("s-task:%s checkpoint-trigger send at:%"PRId64", checkpointId:%"PRId64", transId:%d", pTask->id.idStr,
+          pSendInfo->sendTs, pInfo->activeId, pInfo->transId);
+    }
+
+    taosThreadMutexUnlock(&pInfo->lock);
+    return true;
+  }
+
+  ASSERT(0);
+  return false;
+}
+
+void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal) {
+  *pRecved = taosArrayGetSize(pTask->chkInfo.pActiveInfo->pReadyMsgList);
+
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    *pTotal = 1;
+  } else {
+    *pTotal = streamTaskGetNumOfUpstream(pTask);
+  }
+}
+
+// record the dispatch checkpoint trigger info in the list
+void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+
+  int64_t now = taosGetTimestampMs();
+  taosThreadMutexLock(&pInfo->lock);
+
+  // outputQ should be empty here
+  ASSERT(streamQueueGetNumOfItems(pTask->outputq.queue) == 1);
+
+  pInfo->dispatchTrigger = true;
+  if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
+    STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher;
+
+    STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId};
+    taosArrayPush(pInfo->pDispatchTriggerList, &p);
+  } else {
+    for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) {
+      SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i);
+
+      STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId};
+      taosArrayPush(pInfo->pDispatchTriggerList, &p);
+    }
+  }
+
+  taosThreadMutexUnlock(&pInfo->lock);
+}
+
+int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) {
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+
+  int32_t num = 0;
+  taosThreadMutexLock(&pInfo->lock);
+  for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
+    STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
+    if (p->recved) {
+      num ++;
+    }
+  }
+  taosThreadMutexUnlock(&pInfo->lock);
+  return num;
+}
+
+void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+
+  int32_t taskId = 0;
+
+  taosThreadMutexLock(&pInfo->lock);
+
+  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
+    STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
+    if (p->nodeId == vgId) {
+      ASSERT(p->recved == false);
+
+      p->recved = true;
+      p->recvTs = taosGetTimestampMs();
+      taskId = p->taskId;
+      break;
+    }
+  }
+
+  taosThreadMutexUnlock(&pInfo->lock);
+
+  int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask);
+  int32_t total = streamTaskGetNumOfDownstream(pTask);
+  stDebug("s-task:%s set downstream:0x%x(vgId:%d) checkpoint-trigger dispatch confirmed, total confirmed:%d/%d",
+          pTask->id.idStr, taskId, vgId, numOfConfirmed, total);
+
+  ASSERT(taskId != 0);
+}
+
 static int32_t uploadCheckpointToS3(const char* id, const char* path) {
   TdDirPtr pDir = taosOpenDir(path);
   if (pDir == NULL) return -1;
@@ -576,7 +827,7 @@ int32_t streamTaskUploadCheckpoint(const char* id, const char* path) {
   }
 
   if (strlen(tsSnodeAddress) != 0) {
-    return uploadRsync(id, path);
+    return uploadByRsync(id, path);
   } else if (tsS3StreamEnabled) {
     return uploadCheckpointToS3(id, path);
   }
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 3115c2cb43..c8a626a739 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -23,12 +23,6 @@ typedef struct SBlockName {
   char     parTbName[TSDB_TABLE_NAME_LEN];
 } SBlockName;
 
-typedef struct {
-  int32_t upStreamTaskId;
-  SEpSet  upstreamNodeEpset;
-  SRpcMsg msg;
-} SStreamChkptReadyInfo;
-
 static void    doRetryDispatchData(void* param, void* tmrId);
 static int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet);
 static int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq);
@@ -85,12 +79,14 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
   void*   buf = NULL;
   int32_t sz = taosArrayGetSize(pTask->upstreamInfo.pList);
   ASSERT(sz > 0);
+
   for (int32_t i = 0; i < sz; i++) {
     req->reqId = tGenIdPI64();
-    SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    SStreamUpstreamEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
     req->dstNodeId = pEpInfo->nodeId;
     req->dstTaskId = pEpInfo->taskId;
     int32_t len;
+
     tEncodeSize(tEncodeStreamRetrieveReq, req, len, code);
     if (code != 0) {
       ASSERT(0);
@@ -115,7 +111,6 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
 
     code = tmsgSendReq(&pEpInfo->epSet, &rpcMsg);
     if (code != 0) {
-      ASSERT(0);
       rpcFreeCont(buf);
       return code;
     }
@@ -124,15 +119,16 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
     stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr,
             pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId);
   }
+
   return code;
 }
 
 static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req){
-  SRetrieveTableRsp* pRetrieve = NULL;
   int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
-
-  pRetrieve = taosMemoryCalloc(1, dataStrLen);
-  if (pRetrieve == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  SRetrieveTableRsp* pRetrieve = taosMemoryCalloc(1, dataStrLen);
+  if (pRetrieve == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
 
   int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   pRetrieve->useconds = 0;
@@ -231,6 +227,28 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) {
   pMsgInfo->dispatchMsgType = 0;
 }
 
+int32_t streamTaskBuildAndSendTriggerMsg(SStreamTask* pTask, const SStreamDataBlock* pData, int32_t dstTaskId,
+                                         int32_t vgId, SEpSet* pEpset) {
+  SStreamDispatchReq* pReq = taosMemoryCalloc(1, sizeof(SStreamDispatchReq));
+
+  int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
+  int32_t code = tInitStreamDispatchReq(pReq, pTask, pData->srcVgId, numOfBlocks, dstTaskId, pData->type);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  for (int32_t i = 0; i < numOfBlocks; i++) {
+    SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i);
+    code = streamAddBlockIntoDispatchMsg(pDataBlock, pReq);
+    if (code != TSDB_CODE_SUCCESS) {
+      destroyDispatchMsg(pReq, 1);
+      return code;
+    }
+  }
+
+  return doSendDispatchMsg(pTask, pReq, vgId, pEpset);
+}
+
 static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) {
   int32_t code = 0;
   int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
@@ -357,8 +375,8 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch
     for (int32_t i = 0; i < numOfVgroups; i++) {
       if (pDispatchMsg[i].blockNum > 0) {
         SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
-        stDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr,
-                pTask->info.selfChildId, pDispatchMsg[i].blockNum, pVgInfo->vgId);
+        stDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", id, pTask->info.selfChildId,
+                pDispatchMsg[i].blockNum, pVgInfo->vgId);
 
         code = doSendDispatchMsg(pTask, &pDispatchMsg[i], pVgInfo->vgId, &pVgInfo->epSet);
         if (code < 0) {
@@ -372,8 +390,7 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch
       }
     }
 
-    stDebug("s-task:%s complete shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, numOfVgroups,
-            msgId);
+    stDebug("s-task:%s complete shuffle-dispatch blocks to all %d vnodes, msgId:%d", id, numOfVgroups, msgId);
   }
 
   return code;
@@ -562,7 +579,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
     return 0;
   }
 
-  if (pTask->chkInfo.dispatchCheckpointTrigger) {
+  if (pTask->chkInfo.pActiveInfo->dispatchTrigger) {
     stDebug("s-task:%s already send checkpoint trigger, not dispatch anymore", id);
     atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
     return 0;
@@ -590,6 +607,10 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
   } else {  // todo handle build dispatch msg failed
   }
 
+  if (pBlock->type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
+    streamTaskInitTriggerDispatchInfo(pTask);
+  }
+
   int32_t retryCount = 0;
   while (1) {
     code = sendDispatchMsg(pTask, pTask->msgInfo.pData);
@@ -624,18 +645,24 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
 
 // this function is usually invoked by sink/agg task
 int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
-  int32_t num = taosArrayGetSize(pTask->pReadyMsgList);
+  SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList;
+
+  taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock);
+
+  int32_t num = taosArrayGetSize(pList);
   ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
 
   for (int32_t i = 0; i < num; ++i) {
-    SStreamChkptReadyInfo* pInfo = taosArrayGet(pTask->pReadyMsgList, i);
+    SStreamChkptReadyInfo* pInfo = taosArrayGet(pList, i);
     tmsgSendReq(&pInfo->upstreamNodeEpset, &pInfo->msg);
 
     stDebug("s-task:%s level:%d checkpoint ready msg sent to upstream:0x%x", pTask->id.idStr, pTask->info.taskLevel,
             pInfo->upStreamTaskId);
   }
 
-  taosArrayClear(pTask->pReadyMsgList);
+  taosArrayClear(pList);
+
+  taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
   stDebug("s-task:%s level:%d checkpoint ready msg sent to all %d upstreams", pTask->id.idStr, pTask->info.taskLevel,
           num);
 
@@ -644,21 +671,22 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
 
 // this function is only invoked by source task, and send rsp to mnode
 int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
-  taosThreadMutexLock(&pTask->lock);
+  SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList;
 
+  taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock);
   ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
 
-  if (taosArrayGetSize(pTask->pReadyMsgList) == 1) {
-    SStreamChkptReadyInfo* pInfo = taosArrayGet(pTask->pReadyMsgList, 0);
+  if (taosArrayGetSize(pList) == 1) {
+    SStreamChkptReadyInfo* pInfo = taosArrayGet(pList, 0);
     tmsgSendRsp(&pInfo->msg);
 
-    taosArrayClear(pTask->pReadyMsgList);
+    taosArrayClear(pList);
     stDebug("s-task:%s level:%d source checkpoint completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel);
   } else {
     stDebug("s-task:%s level:%d already send rsp checkpoint success to mnode", pTask->id.idStr, pTask->info.taskLevel);
   }
 
-  taosThreadMutexUnlock(&pTask->lock);
+  taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -777,17 +805,34 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
 }
 
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask) {
-  SStreamChkptReadyInfo info = {0};
+  SStreamChkptReadyInfo info = {
+      .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId};
+
   streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS);
 
-  if (pTask->pReadyMsgList == NULL) {
-    pTask->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo));
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+  taosThreadMutexLock(&pActiveInfo->lock);
+
+  int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList);
+  if (size > 0) {
+    ASSERT(size == 1);
+
+    SStreamChkptReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0);
+    if (pReady->transId == pReq->transId) {
+      stWarn("s-task:%s repeatly recv checkpoint source msg from mnode, checkpointId:%" PRId64 ", ignore",
+             pTask->id.idStr, pReq->checkpointId);
+    } else {
+      stError("s-task:%s checkpointId:%" PRId64 " transId:%d not completed, new transId:%d checkpointId:%" PRId64
+              " recv from mnode",
+              pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId);
+      ASSERT(0); // failed to handle it
+    }
+  } else {
+    taosArrayPush(pActiveInfo->pReadyMsgList, &info);
+    stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size);
   }
 
-  taosArrayPush(pTask->pReadyMsgList, &info);
-
-  int32_t size = taosArrayGetSize(pTask->pReadyMsgList);
-  stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size);
+  taosThreadMutexUnlock(&pActiveInfo->lock);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -799,7 +844,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
     return TSDB_CODE_SUCCESS;
   }
 
-  SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
+  SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
 
   SStreamCheckpointReadyMsg req = {0};
   req.downstreamNodeId = pTask->pMeta->vgId;
@@ -833,7 +878,14 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
 
   ASSERT(req.upstreamTaskId != 0);
 
-  SStreamChkptReadyInfo info = {.upStreamTaskId = pInfo->taskId, .upstreamNodeEpset = pInfo->epSet};
+  SStreamChkptReadyInfo info = {
+      .upStreamTaskId = pInfo->taskId,
+      .upstreamNodeEpset = pInfo->epSet,
+      .nodeId = req.upstreamNodeId,
+      .recvTs = taosGetTimestampMs(),
+      .checkpointId = req.checkpointId,
+  };
+
   initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead));
 
   stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64
@@ -841,39 +893,65 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
           pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index,
           req.upstreamNodeId);
 
-  if (pTask->pReadyMsgList == NULL) {
-    pTask->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo));
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+  taosThreadMutexLock(&pActiveInfo->lock);
+
+  bool recved = false;
+  int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList);
+  for (int32_t i = 0; i < size; ++i) {
+    SStreamChkptReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i);
+    if (p->nodeId == req.upstreamNodeId) {
+      if (p->checkpointId == req.checkpointId) {
+        stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 ", ignore",
+               pTask->id.idStr, p->upStreamTaskId, p->nodeId, p->checkpointId);
+      } else {
+        stError("s-task:%s checkpointId:%" PRId64 " not completed, new checkpointId:%" PRId64 " recv",
+                pTask->id.idStr, p->checkpointId, checkpointId);
+        ASSERT(0); // failed to handle it
+      }
+
+      recved = true;
+      break;
+    }
   }
 
-  taosArrayPush(pTask->pReadyMsgList, &info);
+  if (!recved) {
+    taosArrayPush(pActiveInfo->pReadyMsgList, &info);
+  }
+
+  taosThreadMutexUnlock(&pActiveInfo->lock);
   return 0;
 }
 
 void streamClearChkptReadyMsg(SStreamTask* pTask) {
-  if (pTask->pReadyMsgList == NULL) {
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+  if (pActiveInfo == NULL) {
     return;
   }
 
-  for (int i = 0; i < taosArrayGetSize(pTask->pReadyMsgList); i++) {
-    SStreamChkptReadyInfo* pInfo = taosArrayGet(pTask->pReadyMsgList, i);
+  for (int i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); i++) {
+    SStreamChkptReadyInfo* pInfo = taosArrayGet(pActiveInfo->pReadyMsgList, i);
     rpcFreeCont(pInfo->msg.pCont);
   }
-  taosArrayClear(pTask->pReadyMsgList);
+
+  taosArrayClear(pActiveInfo->pReadyMsgList);
 }
 
 // this message has been sent successfully, let's try next one.
-static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) {
+static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId, int32_t downstreamNodeId) {
   stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData);
 
   bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
   if (delayDispatch) {
     taosThreadMutexLock(&pTask->lock);
     // we only set the dispatch msg info for current checkpoint trans
-    if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK && pTask->chkInfo.checkpointingId == pTask->msgInfo.checkpointId) {
-      ASSERT(pTask->chkInfo.transId == pTask->msgInfo.transId);
-      pTask->chkInfo.dispatchCheckpointTrigger = true;
-      stDebug("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 " transId:%d confirmed",
-             pTask->id.idStr, pTask->msgInfo.checkpointId, pTask->msgInfo.transId);
+    if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK &&
+        pTask->chkInfo.pActiveInfo->activeId == pTask->msgInfo.checkpointId) {
+      ASSERT(pTask->chkInfo.pActiveInfo->transId == pTask->msgInfo.transId);
+      stDebug("s-task:%s checkpoint-trigger msg to 0x%x rsp for checkpointId:%" PRId64 " transId:%d confirmed",
+             pTask->id.idStr, downstreamId, pTask->msgInfo.checkpointId, pTask->msgInfo.transId);
+
+      streamTaskSetTriggerDispatchConfirmed(pTask, downstreamNodeId);
     } else {
       stWarn("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 " transId:%d discard, since expired",
              pTask->id.idStr, pTask->msgInfo.checkpointId, pTask->msgInfo.transId);
@@ -966,10 +1044,10 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
           pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
         stError("s-task:%s failed to dispatch checkpoint-trigger msg, checkpointId:%" PRId64
                 ", set the current checkpoint failed, and send rsp to mnode",
-                id, pTask->chkInfo.checkpointingId);
+                id, pTask->chkInfo.pActiveInfo->activeId);
         { // send checkpoint failure msg to mnode directly
-          pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId;   // record the latest failed checkpoint id
-          pTask->chkInfo.checkpointingId = pTask->chkInfo.checkpointingId;
+          pTask->chkInfo.pActiveInfo->failedId = pTask->chkInfo.pActiveInfo->activeId;   // record the latest failed checkpoint id
+          pTask->chkInfo.pActiveInfo->activeId = pTask->chkInfo.pActiveInfo->activeId;
           streamTaskSendCheckpointSourceRsp(pTask);
         }
       } else {
@@ -1035,7 +1113,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
         // now ready for next data output
         atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
       } else {
-        handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId);
+        handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
       }
     }
   }
@@ -1096,7 +1174,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
   stDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64 ", msgId:%d", id,
           pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId);
 
-  SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId);
+  SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId);
   ASSERT(pInfo != NULL);
 
   if (pMeta->role == NODE_ROLE_FOLLOWER) {
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index b60164fca9..87f239f31c 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -426,7 +426,7 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) {
         streamMetaReleaseTask(pMeta, pStreamTask);
         return code;
       } else {
-        stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr);
+        stDebug("s-task:%s sink task halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr);
       }
       streamMetaReleaseTask(pMeta, pStreamTask);
     }
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 00fe1207dd..7356dc01ce 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1044,13 +1044,13 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
       entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize);
     }
 
-    if ((*pTask)->chkInfo.checkpointingId != 0) {
-      entry.checkpointInfo.failed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId) ? 1 : 0;
-      entry.checkpointInfo.activeId = (*pTask)->chkInfo.checkpointingId;
-      entry.checkpointInfo.activeTransId = (*pTask)->chkInfo.transId;
+    if ((*pTask)->chkInfo.pActiveInfo->activeId != 0) {
+      entry.checkpointInfo.failed = ((*pTask)->chkInfo.pActiveInfo->failedId >= (*pTask)->chkInfo.pActiveInfo->activeId) ? 1 : 0;
+      entry.checkpointInfo.activeId = (*pTask)->chkInfo.pActiveInfo->activeId;
+      entry.checkpointInfo.activeTransId = (*pTask)->chkInfo.pActiveInfo->transId;
 
       if (entry.checkpointInfo.failed) {
-        stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId);
+        stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.pActiveInfo->transId);
       }
     }
 
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index c056e2a4b6..e0415c8467 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -24,6 +24,8 @@
 static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo);
 static void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated);
 static void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdate);
+static void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo);
+static SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo();
 
 static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) {
   int32_t childId = taosArrayGetSize(pArray);
@@ -70,12 +72,12 @@ static void freeItem(void* p) {
 }
 
 static void freeUpstreamItem(void* p) {
-  SStreamChildEpInfo** pInfo = p;
+  SStreamUpstreamEpInfo** pInfo = p;
   taosMemoryFree(*pInfo);
 }
 
-static SStreamChildEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) {
-  SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo));
+static SStreamUpstreamEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) {
+  SStreamUpstreamEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamUpstreamEpInfo));
   if (pEpInfo == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
@@ -254,7 +256,6 @@ void tFreeStreamTask(SStreamTask* pTask) {
   }
 
   streamClearChkptReadyMsg(pTask);
-  pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList);
 
   if (pTask->msgInfo.pData != NULL) {
     clearBufferedDispatchMsg(pTask);
@@ -302,6 +303,9 @@ void tFreeStreamTask(SStreamTask* pTask) {
     taosMemoryFree((void*)pTask->id.idStr);
   }
 
+  streamTaskDestroyActiveChkptInfo(pTask->chkInfo.pActiveInfo);
+  pTask->chkInfo.pActiveInfo = NULL;
+
   taosMemoryFree(pTask);
   stDebug("s-task:0x%x free task completed", taskId);
 }
@@ -414,6 +418,10 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  if (pTask->chkInfo.pActiveInfo == NULL) {
+    pTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo();
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -433,8 +441,12 @@ int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) {
   }
 }
 
+int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask) {
+  return taosArrayGetSize(pTask->upstreamInfo.pList);
+}
+
 int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask) {
-  SStreamChildEpInfo* pEpInfo = createStreamTaskEpInfo(pUpstreamTask);
+  SStreamUpstreamEpInfo* pEpInfo = createStreamTaskEpInfo(pUpstreamTask);
   if (pEpInfo == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -453,7 +465,7 @@ void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpS
 
   int32_t numOfUpstream = taosArrayGetSize(pTask->upstreamInfo.pList);
   for (int32_t i = 0; i < numOfUpstream; ++i) {
-    SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
     if (pInfo->nodeId == nodeId) {
       bool equal = isEpsetEqual(&pInfo->epSet, pEpSet);
       if (!equal) {
@@ -589,7 +601,7 @@ void streamTaskResetUpstreamStageInfo(SStreamTask* pTask) {
 
   int32_t size = taosArrayGetSize(pTask->upstreamInfo.pList);
   for (int32_t i = 0; i < size; ++i) {
-    SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
     pInfo->stage = -1;
   }
 
@@ -603,7 +615,7 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) {
   }
 
   for (int32_t i = 0; i < num; ++i) {
-    SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
     pInfo->dataAllowed = true;
   }
 
@@ -612,12 +624,19 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) {
 }
 
 void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
-  SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId);
+  SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId);
   if (pInfo != NULL) {
     pInfo->dataAllowed = false;
   }
 }
 
+void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) {
+  SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId);
+  if (pInfo != NULL) {
+    pInfo->dataAllowed = true;
+  }
+}
+
 bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask) {
   return pTask->upstreamInfo.numOfClosed == taosArrayGetSize(pTask->upstreamInfo.pList);
 }
@@ -723,9 +742,9 @@ int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStr
   pReq->dropRelHTask = dropRelHTask;
   pReq->hStreamId = pHTaskId->streamId;
   pReq->hTaskId = pHTaskId->taskId;
-  pReq->transId = pCheckpointInfo->transId;
+  pReq->transId = pCheckpointInfo->pActiveInfo->transId;
 
-  pReq->checkpointId = pCheckpointInfo->checkpointingId;
+  pReq->checkpointId = pCheckpointInfo->pActiveInfo->activeId;
   pReq->checkpointVer = pCheckpointInfo->processedVer;
   pReq->checkpointTs = pCheckpointInfo->startTs;
 
@@ -860,10 +879,10 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
   return 0;
 }
 
-SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) {
+SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) {
   int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
   for (int32_t i = 0; i < num; ++i) {
-    SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
     if (pInfo->taskId == taskId) {
       return pInfo;
     }
@@ -873,6 +892,24 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t
   return NULL;
 }
 
+SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) {
+  if (pTask->info.taskLevel == TASK_OUTPUT__FIXED_DISPATCH) {
+    if (pTask->outputInfo.fixedDispatcher.taskId == taskId) {
+      return &pTask->outputInfo.fixedDispatcher.epSet;
+    }
+  } else if (pTask->info.taskLevel == TASK_OUTPUT__SHUFFLE_DISPATCH) {
+    SArray* pList = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
+    for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+      SVgroupInfo* pVgInfo = taosArrayGet(pList, i);
+      if (pVgInfo->taskId == taskId) {
+        return &pVgInfo->epSet;
+      }
+    }
+  }
+
+  return NULL;
+}
+
 char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) {
   char buf[128] = {0};
   sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId);
@@ -914,4 +951,64 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) {
 
 void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) {
   pTask->status.removeBackendFiles = true;
+}
+
+int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) {
+  if (pTransId != NULL) {
+    *pTransId = pTask->chkInfo.pActiveInfo->transId;
+  }
+
+  if (pCheckpointId != NULL) {
+    *pCheckpointId = pTask->chkInfo.pActiveInfo->activeId;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId) {
+  pTask->chkInfo.pActiveInfo->activeId = activeCheckpointId;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId) {
+  pTask->chkInfo.pActiveInfo->transId = transId;
+  pTask->chkInfo.pActiveInfo->activeId = checkpointId;
+  pTask->chkInfo.pActiveInfo->failedId = checkpointId;
+  return TSDB_CODE_SUCCESS;
+}
+
+SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo() {
+  SActiveCheckpointInfo* pInfo = taosMemoryCalloc(1, sizeof(SActiveCheckpointInfo));
+  taosThreadMutexInit(&pInfo->lock, NULL);
+
+  pInfo->pDispatchTriggerList = taosArrayInit(4, sizeof(STaskTriggerSendInfo));
+  pInfo->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo));
+  return pInfo;
+}
+
+void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
+  if (pInfo == NULL) {
+    return;
+  }
+
+  taosThreadMutexDestroy(&pInfo->lock);
+  pInfo->pDispatchTriggerList = taosArrayDestroy(pInfo->pDispatchTriggerList);
+  pInfo->pReadyMsgList = taosArrayDestroy(pInfo->pReadyMsgList);
+
+  if (pInfo->pCheckTmr != NULL) {
+    taosTmrStop(pInfo->pCheckTmr);
+    pInfo->pCheckTmr = NULL;
+  }
+
+  taosMemoryFree(pInfo);
+}
+
+void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) {
+  pInfo->activeId = 0;  // clear the checkpoint id
+  pInfo->failedId = 0;
+  pInfo->transId = 0;
+  pInfo->dispatchTrigger = false;
+
+  taosArrayClear(pInfo->pReadyMsgList);
+  taosArrayClear(pInfo->pDispatchTriggerList);
 }
\ No newline at end of file
diff --git a/source/libs/stream/src/streammsg.c b/source/libs/stream/src/streammsg.c
index 9b69833234..f8228a8f5f 100644
--- a/source/libs/stream/src/streammsg.c
+++ b/source/libs/stream/src/streammsg.c
@@ -17,7 +17,7 @@
 #include "streammsg.h"
 #include "tstream.h"
 
-int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) {
+int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) {
   if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1;
   if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1;
   if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1;
@@ -26,7 +26,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo)
   return 0;
 }
 
-int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
+int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamUpstreamEpInfo* pInfo) {
   if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1;
@@ -481,7 +481,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList);
   if (tEncodeI32(pEncoder, epSz) < 0) return -1;
   for (int32_t i = 0; i < epSz; i++) {
-    SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
     if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1;
   }
 
@@ -557,7 +557,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
 
   pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES);
   for (int32_t i = 0; i < epSz; i++) {
-    SStreamChildEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamChildEpInfo));
+    SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo));
     if (pInfo == NULL) return -1;
     if (tDecodeStreamEpInfo(pDecoder, pInfo) < 0) {
       taosMemoryFreeClear(pInfo);

From 7fc2e8cc26ab977aa41c8cecedd11266cd593779 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 28 May 2024 10:17:20 +0800
Subject: [PATCH 043/132] batch meta

---
 source/client/src/clientRawBlockWrite.c | 12 ++++-
 source/client/src/clientTmq.c           | 45 +++++++++++++---
 source/dnode/vnode/src/tq/tqUtil.c      | 68 ++++++++++++++++++++-----
 source/libs/executor/src/scanoperator.c |  3 +-
 utils/test/c/tmq_taosx_ci.c             |  6 +--
 5 files changed, 107 insertions(+), 27 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index a37cd99360..d5f478d188 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -2051,6 +2051,7 @@ static char* processSimpleMeta(SMqMetaRsp* pMetaRsp) {
 static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) {
   SDecoder        coder;
   SMqBatchMetaRsp rsp = {0};
+  SArray*         strArray = NULL;
   tDecoderInit(&coder, pMsgRsp->pMetaBuff, pMsgRsp->metaBuffLen);
   if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) {
     goto _end;
@@ -2058,7 +2059,7 @@ static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) {
 
   int64_t fullSize = 0;
   int32_t num = taosArrayGetSize(rsp.batchMetaReq);
-  SArray* strArray = taosArrayInit(num, POINTER_BYTES);
+  strArray = taosArrayInit(num, POINTER_BYTES);
   for (int32_t i = 0; i < num; i++) {
     int32_t len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i);
     void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
@@ -2078,10 +2079,17 @@ static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) {
   for (int32_t i = 0; i < num; i++) {
     char* subStr = taosArrayGetP(strArray, i);
     strcat(buf, subStr);
-    strcat(buf, "\n");
+    if (i < num - 1) {
+      strcat(buf, "\n");
+    }
   }
+  taosArrayDestroyP(strArray, taosMemoryFree);
+  tDeleteMqBatchMetaRsp(&rsp);
+  return buf;
 
 _end:
+  taosArrayDestroyP(strArray, taosMemoryFree);
+  tDeleteMqBatchMetaRsp(&rsp);
   return NULL;
 }
 
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 3fbae1b774..69a4503680 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -2009,7 +2009,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
         tmqFreeRspWrapper(pRspWrapper);
         taosFreeQitem(pRspWrapper);
       }
-    } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
+    } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
       int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
 
@@ -2032,12 +2032,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
         updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset,
                      pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true);
         // build rsp
-        void* pRsp = NULL;
-        if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
-          pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
-        } else {
-          pRsp = tmqBuildBatchMetaRspFromWrapper(pollRspWrapper);
-        }
+        SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper);
         taosFreeQitem(pRspWrapper);
         taosWUnLockLatch(&tmq->lock);
         return pRsp;
@@ -2048,6 +2043,42 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
         tmqFreeRspWrapper(pRspWrapper);
         taosFreeQitem(pRspWrapper);
       }
+    } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
+      SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
+      int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
+
+      tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId);
+
+      if (pollRspWrapper->batchMetaRsp.head.epoch == consumerEpoch) {
+        taosWLockLatch(&tmq->lock);
+        SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
+        pollRspWrapper->vgHandle = pVg;
+        pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
+        if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) {
+          tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
+                   pollRspWrapper->topicName, pollRspWrapper->vgId);
+          tmqFreeRspWrapper(pRspWrapper);
+          taosFreeQitem(pRspWrapper);
+          taosWUnLockLatch(&tmq->lock);
+          return NULL;
+        }
+
+        // build rsp
+        void* pRsp = NULL;
+        updateVgInfo(pVg, &pollRspWrapper->batchMetaRsp.rspOffset, &pollRspWrapper->batchMetaRsp.rspOffset,
+                     pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever,
+                     tmq->consumerId, true);
+        pRsp = tmqBuildBatchMetaRspFromWrapper(pollRspWrapper);
+        taosFreeQitem(pRspWrapper);
+        taosWUnLockLatch(&tmq->lock);
+        return pRsp;
+      } else {
+        tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
+                tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->batchMetaRsp.head.epoch, consumerEpoch);
+        setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
+        tmqFreeRspWrapper(pRspWrapper);
+        taosFreeQitem(pRspWrapper);
+      }
     } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
       int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index df7539bb69..88c41b99ef 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -191,14 +191,13 @@ end : {
 
 static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
                                                   SRpcMsg* pMsg, STqOffsetVal* offset) {
-  int        code = 0;
-  int32_t    vgId = TD_VID(pTq->pVnode);
-  SMqMetaRsp metaRsp = {0};
-  STaosxRsp  taosxRsp = {0};
+  int             code = 0;
+  int32_t         vgId = TD_VID(pTq->pVnode);
+  STaosxRsp       taosxRsp = {0};
+  SMqBatchMetaRsp btMetaRsp = {0};
   tqInitTaosxRsp(&taosxRsp.common, *offset);
 
   if (offset->type != TMQ_OFFSET__LOG) {
-    SMqBatchMetaRsp btMetaRsp = {0};
     if (tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset) < 0) {
       code = -1;
       goto end;
@@ -232,11 +231,18 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
 
     uint64_t st = taosGetTimestampMs();
     int      totalRows = 0;
+    int32_t  totalMetaRows = 0;
     while (1) {
       int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
       ASSERT(savedEpoch <= pRequest->epoch);
 
       if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) {
+        if (totalMetaRows > 0) {
+          tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
+          tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
+          ASSERT(totalRows == 0);
+          goto end;
+        }
         tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer);
         code = tqSendDataRsp(
             pHandle, pMsg, pRequest, &taosxRsp,
@@ -270,12 +276,48 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
           }
         }
 
-        tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
-        tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
-        metaRsp.resMsgType = pHead->msgType;
-        metaRsp.metaRspLen = pHead->bodyLen;
-        metaRsp.metaRsp = pHead->body;
-        code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId);
+        tqDebug("fetch meta msg, ver:%" PRId64 ", vgId:%d, type:%s", pHead->version, vgId, TMSG_INFO(pHead->msgType));
+        if (!btMetaRsp.batchMetaReq) {
+          btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES);
+          btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t));
+        }
+        fetchVer++;
+
+        SMqMetaRsp tmpMetaRsp = {0};
+        tmpMetaRsp.resMsgType = pHead->msgType;
+        tmpMetaRsp.metaRspLen = pHead->bodyLen;
+        tmpMetaRsp.metaRsp = pHead->body;
+        uint32_t len = 0;
+        tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code);
+        if (TSDB_CODE_SUCCESS != code) {
+          tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
+          continue;
+        }
+        int32_t tLen = sizeof(SMqRspHead) + len;
+        void*   tBuf = taosMemoryCalloc(1, tLen);
+        void*   metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
+        SEncoder encoder = {0};
+        tEncoderInit(&encoder, metaBuff, len);
+        code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
+        if (code < 0) {
+          tEncoderClear(&encoder);
+          tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
+          continue;
+        }
+        taosArrayPush(btMetaRsp.batchMetaReq, &tBuf);
+        taosArrayPush(btMetaRsp.batchMetaLen, &tLen);
+        totalMetaRows++;
+        if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > 1000)) {
+          tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
+          tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
+          goto end;
+        }
+        continue;
+      }
+
+      if (totalMetaRows > 0) {
+        tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
+        tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
         goto end;
       }
 
@@ -382,8 +424,8 @@ int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SM
   SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
 
   tmsgSendRsp(&resp);
-  tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, offset type:%d", vgId,
-          pReq->consumerId, pReq->epoch, pRsp->rspOffset.type);
+  tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%d offset type:%d", vgId,
+          pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type);
 
   return 0;
 }
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 72ea9afae7..851d2f2735 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -44,7 +44,6 @@ int32_t scanDebug = 0;
 #define STREAM_SCAN_OP_NAME               "StreamScanOperator"
 #define STREAM_SCAN_OP_STATE_NAME         "StreamScanFillHistoryState"
 #define STREAM_SCAN_OP_CHECKPOINT_NAME    "StreamScanOperator_Checkpoint"
-#define TMQ_MAX_BATCH_SIZE                4096
 
 typedef struct STableMergeScanExecInfo {
   SFileBlockLoadRecorder blockRecorder;
@@ -2894,7 +2893,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
     return NULL;
   } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) {
     SSnapContext* sContext = pInfo->sContext;
-    for(int32_t i = 0; i < TMQ_MAX_BATCH_SIZE; i++) {
+    for(int32_t i = 0; i < tmqRowSize; i++) {
       void*         data = NULL;
       int32_t       dataLen = 0;
       int16_t       type = 0;
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 5642b801f4..ef2d70f54f 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -1064,7 +1064,7 @@ void testConsumeExcluded(int topic_type) {
     char* topic = "create topic topic_excluded with meta as database db_taosx";
     pRes = taos_query(pConn, topic);
     if (taos_errno(pRes) != 0) {
-      printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes));
+      printf("failed to create topic topic_excluded1, reason:%s\n", taos_errstr(pRes));
       taos_close(pConn);
       return;
     }
@@ -1073,7 +1073,7 @@ void testConsumeExcluded(int topic_type) {
     char* topic = "create topic topic_excluded as select * from stt";
     pRes = taos_query(pConn, topic);
     if (taos_errno(pRes) != 0) {
-      printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes));
+      printf("failed to create topic topic_excluded2, reason:%s\n", taos_errstr(pRes));
       taos_close(pConn);
       return;
     }
@@ -1115,7 +1115,7 @@ void testConsumeExcluded(int topic_type) {
         assert(raw.raw_type != 2 && raw.raw_type != 4 && raw.raw_type != TDMT_VND_CREATE_STB &&
                raw.raw_type != TDMT_VND_ALTER_STB && raw.raw_type != TDMT_VND_CREATE_TABLE &&
                raw.raw_type != TDMT_VND_ALTER_TABLE && raw.raw_type != TDMT_VND_DELETE);
-        assert(raw.raw_type == TDMT_VND_DROP_STB || raw.raw_type == TDMT_VND_DROP_TABLE);
+        assert(raw.raw_type == TDMT_VND_DROP_STB || raw.raw_type == TDMT_VND_DROP_TABLE || raw.raw_type == 5);
       } else if (topic_type == 2) {
         assert(0);
       }

From 57bf9d10482a7a5c36bfade104540758e4ed6de4 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 28 May 2024 10:56:43 +0800
Subject: [PATCH 044/132] adj log

---
 source/dnode/vnode/src/tq/tqUtil.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 88c41b99ef..4b1ff04331 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -424,7 +424,7 @@ int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SM
   SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
 
   tmsgSendRsp(&resp);
-  tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%d offset type:%d", vgId,
+  tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%ld offset type:%d", vgId,
           pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type);
 
   return 0;

From cc318d7e2d7da8491558a64e7f7569cfb30d2898 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 28 May 2024 11:34:04 +0800
Subject: [PATCH 045/132] enh(stream): sink task does not initialize the
 backend.

---
 source/dnode/vnode/src/tqCommon/tqCommon.c    | 15 +++++++++------
 source/libs/stream/src/streamBackendRocksdb.c |  2 +-
 source/libs/stream/src/streamMeta.c           |  3 +--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index b9c0589dc5..a137086d21 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -51,12 +51,15 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
     taskId = replaceStreamTaskId(pTask);
   }
 
-  pTask->pState = streamStateOpen(pMeta->path, pTask, false, -1, -1);
-  if (pTask->pState == NULL) {
-    tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
-    return -1;
-  } else {
-    tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
+  // sink task does not need the pState
+  if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
+    pTask->pState = streamStateOpen(pMeta->path, pTask, false, -1, -1);
+    if (pTask->pState == NULL) {
+      tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
+      return -1;
+    } else {
+      tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
+    }
   }
 
   if (pTask->info.fillHistory) {
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 87fb615d5f..d42a3b545a 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -338,7 +338,7 @@ int32_t rebuildFromRemoteChkp_rsync(const char* key, char* chkptPath, int64_t ch
   if (taosIsDir(defaultPath)) {
     taosRemoveDir(defaultPath);
     taosMulMkDir(defaultPath);
-    stDebug("clear local default dir before download checkpoint data:%s succ", defaultPath);
+    stDebug("clear local default dir before downloading checkpoint data:%s succ", defaultPath);
   }
 
   code = streamTaskDownloadCheckpointData(key, chkptPath);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 51a6542256..e0822f60e7 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -250,7 +250,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
 
   taosThreadMutexLock(&pMeta->backendMutex);
   void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key));
-  if (ppBackend != NULL && *ppBackend != NULL) {
+  if ((ppBackend != NULL) && (*ppBackend != NULL)) {
     taskDbAddRef(*ppBackend);
 
     STaskDbWrapper* pBackend = *ppBackend;
@@ -258,7 +258,6 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
     pTask->pBackend = pBackend;
 
     taosThreadMutexUnlock(&pMeta->backendMutex);
-
     stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
     return 0;
   }

From 242b836eca1f2a7833282ce4cadea3199c27749e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 28 May 2024 14:35:58 +0800
Subject: [PATCH 046/132] compile error

---
 source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 388bc6aef3..cbac6a13de 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -2355,8 +2355,6 @@ _OVER:
 }
 
 #ifndef TD_ENTERPRISE
-int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  return 0;
-}
+int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; }
 int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; }
 #endif

From a8383369ba3959eb77b6069bfefdb2489fdf6874 Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Tue, 28 May 2024 15:13:58 +0800
Subject: [PATCH 047/132] enh: arb check roletime before check sync

---
 include/dnode/mnode/mnode.h               |  2 ++
 source/dnode/mnode/impl/src/mndArbGroup.c |  9 ++++++-
 source/dnode/mnode/impl/src/mndMain.c     | 31 ++++++++++++++---------
 3 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h
index 108e6f18a6..fe96fe1117 100644
--- a/include/dnode/mnode/mnode.h
+++ b/include/dnode/mnode/mnode.h
@@ -101,6 +101,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
  */
 int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
 
+int64_t mndGetRoleTimeMs(SMnode *pMnode);
+
 /**
  * @brief Process the rpc, sync request.
  *
diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c
index 50338fe889..b00da9ba3f 100644
--- a/source/dnode/mnode/impl/src/mndArbGroup.c
+++ b/source/dnode/mnode/impl/src/mndArbGroup.c
@@ -540,6 +540,14 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
     return -1;
   }
 
+  int64_t roleTimeMs = mndGetRoleTimeMs(pMnode);
+  int64_t nowMs = taosGetTimestampMs();
+  if (nowMs - roleTimeMs < tsArbHeartBeatIntervalSec * 1000 * 2) {
+    mInfo("arb skip to check sync since mnd had just switch over, roleTime:%" PRId64 " now:%" PRId64, roleTimeMs,
+          nowMs);
+    return 0;
+  }
+
   SArray *pUpdateArray = taosArrayInit(16, sizeof(SArbGroup));
 
   while (1) {
@@ -551,7 +559,6 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
     taosThreadMutexUnlock(&pArbGroup->mutex);
 
     int32_t vgId = arbGroupDup.vgId;
-    int64_t nowMs = taosGetTimestampMs();
 
     bool                member0IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 0, nowMs);
     bool                member1IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 1, nowMs);
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index a78edcb05e..850c527a14 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -334,6 +334,8 @@ static int32_t minCronTime() {
   min = TMIN(min, tsStreamCheckpointInterval);
   min = TMIN(min, 6);  // checkpointRemain
   min = TMIN(min, tsStreamNodeCheckInterval);
+  min = TMIN(min, tsArbHeartBeatIntervalSec);
+  min = TMIN(min, tsArbCheckSyncIntervalSec);
 
   int64_t telemInt = TMIN(60, (tsTelemInterval - 1));
   min = TMIN(min, telemInt);
@@ -390,6 +392,18 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
   if (sec % tsUptimeInterval == 0) {
     mndIncreaseUpTime(pMnode);
   }
+
+  if (sec % (tsArbHeartBeatIntervalSec) == 0) {
+    if (mndPullupArbHeartbeat(pMnode) != 0) {
+      mError("failed to pullup arb heartbeat, since:%s", terrstr());
+    }
+  }
+
+  if (sec % (tsArbCheckSyncIntervalSec) == 0) {
+    if (mndPullupArbCheckSync(pMnode) != 0) {
+      mError("failed to pullup arb check sync, since:%s", terrstr());
+    }
+  }
 }
 void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) {
   if (sec % (tsStatusInterval * 5) == 0) {
@@ -421,18 +435,6 @@ static void *mndThreadFp(void *param) {
       continue;
     }
     mndDoTimerPullupTask(pMnode, sec);
-
-    if (sec % (tsArbHeartBeatIntervalSec) == 0) {
-      if (mndPullupArbHeartbeat(pMnode) != 0) {
-        mError("failed to pullup arb heartbeat, since:%s", terrstr());
-      }
-    }
-
-    if (sec % (tsArbCheckSyncIntervalSec) == 0) {
-      if (mndPullupArbCheckSync(pMnode) != 0) {
-        mError("failed to pullup arb check sync, since:%s", terrstr());
-      }
-    }
   }
 
   return NULL;
@@ -1076,6 +1078,11 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
   return 0;
 }
 
+int64_t mndGetRoleTimeMs(SMnode *pMnode) {
+  SSyncState state = syncGetState(pMnode->syncMgmt.sync);
+  return state.roleTimeMs;
+}
+
 void mndSetRestored(SMnode *pMnode, bool restored) {
   if (restored) {
     taosThreadRwlockWrlock(&pMnode->lock);

From 4c47626baf315221d36e5c407ae149d79aad39dd Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Tue, 28 May 2024 16:29:18 +0800
Subject: [PATCH 048/132] fix: arb distinguish between isSync and acked

---
 include/common/tmsg.h                     | 19 +++++++++-----
 source/common/src/systable.c              |  1 +
 source/common/src/tmsg.c                  | 15 +++++++++++
 source/dnode/mnode/impl/inc/mndDef.h      |  1 +
 source/dnode/mnode/impl/src/mndArbGroup.c | 32 ++++++++++++++++-------
 5 files changed, 53 insertions(+), 15 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index be2f443140..6aa2cd8c36 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2361,12 +2361,19 @@ typedef struct {
 } SMArbUpdateGroupMember;
 
 typedef struct {
-  int32_t                vgId;
-  int64_t                dbUid;
-  SMArbUpdateGroupMember members[2];
-  int8_t                 isSync;
-  SMArbUpdateGroupMember assignedLeader;
-  int64_t                version;
+  int32_t dnodeId;
+  char*   token;
+  int8_t  acked;
+} SMArbUpdateGroupAssigned;
+
+typedef struct {
+  int32_t                  vgId;
+  int64_t                  dbUid;
+  SMArbUpdateGroupMember   members[2];
+  int8_t                   isSync;
+  int8_t                   assignedAcked;
+  SMArbUpdateGroupAssigned assignedLeader;
+  int64_t                  version;
 } SMArbUpdateGroup;
 
 typedef struct {
diff --git a/source/common/src/systable.c b/source/common/src/systable.c
index 9de682dd3a..95b18705cd 100644
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@@ -76,6 +76,7 @@ static const SSysDbTableSchema arbGroupsSchema[] = {
     {.name = "is_sync", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "assigned_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "assigned_token", .bytes = TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "assigned_acked", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
 };
 
 static const SSysDbTableSchema clusterSchema[] = {
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index ef37a41fcf..ad1243f21b 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -6460,6 +6460,11 @@ int32_t tSerializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdat
     if (tEncodeI64(&encoder, pGroup->version) < 0) return -1;
   }
 
+  for (int32_t i = 0; i < sz; i++) {
+    SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
+    if (tEncodeI8(&encoder, pGroup->assignedLeader.acked) < 0) return -1;
+  }
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -6492,8 +6497,18 @@ int32_t tDeserializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpd
     group.assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
     if (tDecodeCStrTo(&decoder, group.assignedLeader.token) < 0) return -1;
     if (tDecodeI64(&decoder, &group.version) < 0) return -1;
+    group.assignedLeader.acked = false;
+
     taosArrayPush(updateArray, &group);
   }
+
+  if (!tDecodeIsEnd(&decoder)) {
+    for (int32_t i = 0; i < sz; i++) {
+      SMArbUpdateGroup *pGroup = taosArrayGet(updateArray, i);
+      if (tDecodeI8(&decoder, &pGroup->assignedLeader.acked) < 0) return -1;
+    }
+  }
+
   pReq->updateArray = updateArray;
 
   tEndDecode(&decoder);
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index 0c40622d08..5c21e9b22b 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -255,6 +255,7 @@ typedef struct {
 typedef struct {
   int32_t dnodeId;
   char    token[TSDB_ARB_TOKEN_SIZE];
+  int8_t  acked;
 } SArbAssignedLeader;
 
 typedef struct {
diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c
index b00da9ba3f..6a6b3d2daa 100644
--- a/source/dnode/mnode/impl/src/mndArbGroup.c
+++ b/source/dnode/mnode/impl/src/mndArbGroup.c
@@ -25,7 +25,7 @@
 #include "mndVgroup.h"
 
 #define ARBGROUP_VER_NUMBER   1
-#define ARBGROUP_RESERVE_SIZE 64
+#define ARBGROUP_RESERVE_SIZE 63
 
 static SHashObj *arbUpdateHash = NULL;
 
@@ -129,6 +129,7 @@ SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup) {
   SDB_SET_INT32(pRaw, dataPos, pLeader->dnodeId, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pGroup->version, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pLeader->acked, _OVER)
 
   SDB_SET_RESERVE(pRaw, dataPos, ARBGROUP_RESERVE_SIZE, _OVER)
 
@@ -182,6 +183,7 @@ SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pLeader->dnodeId, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pGroup->version, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &pLeader->acked, _OVER)
 
   pGroup->mutexInited = false;
 
@@ -235,6 +237,7 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p
   pOld->isSync = pNew->isSync;
   pOld->assignedLeader.dnodeId = pNew->assignedLeader.dnodeId;
   memcpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+  pOld->assignedLeader.acked = pNew->assignedLeader.acked;
   pOld->version++;
 
 _OVER:
@@ -565,8 +568,8 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
     SArbAssignedLeader *pAssignedLeader = &arbGroupDup.assignedLeader;
     int32_t             currentAssignedDnodeId = pAssignedLeader->dnodeId;
 
-    // 1. has assigned && is sync => send req
-    if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true) {
+    // 1. has assigned && is sync && no response => send req
+    if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true && pAssignedLeader->acked == false) {
       (void)mndSendArbSetAssignedLeaderReq(pMnode, currentAssignedDnodeId, vgId, arbToken, term,
                                            pAssignedLeader->token);
       mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, currentAssignedDnodeId);
@@ -658,6 +661,7 @@ static void mndInitArbUpdateGroup(SArbGroup *pGroup, SMArbUpdateGroup *outGroup)
   outGroup->isSync = pGroup->isSync;
   outGroup->assignedLeader.dnodeId = pGroup->assignedLeader.dnodeId;
   outGroup->assignedLeader.token = pGroup->assignedLeader.token;  // just copy the pointer
+  outGroup->assignedLeader.acked = pGroup->assignedLeader.acked;
   outGroup->version = pGroup->version;
 }
 
@@ -773,6 +777,7 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
     newGroup.isSync = pUpdateGroup->isSync;
     newGroup.assignedLeader.dnodeId = pUpdateGroup->assignedLeader.dnodeId;
     memcpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+    newGroup.assignedLeader.acked = pUpdateGroup->assignedLeader.acked;
     newGroup.version = pUpdateGroup->version;
 
     SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId);
@@ -790,10 +795,10 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
       goto _OVER;
     }
 
-    mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s]",
+    mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]",
           pTrans->id, newGroup.vgId, newGroup.members[0].info.dnodeId, newGroup.members[0].state.token,
           newGroup.members[1].info.dnodeId, newGroup.members[1].state.token, newGroup.isSync,
-          newGroup.assignedLeader.dnodeId, newGroup.assignedLeader.token);
+          newGroup.assignedLeader.dnodeId, newGroup.assignedLeader.token, newGroup.assignedLeader.acked);
 
     sdbRelease(pMnode->pSdb, pOldGroup);
   }
@@ -826,11 +831,13 @@ static void mndArbGroupSetAssignedLeader(SArbGroup *pGroup, int32_t index) {
 
   pGroup->assignedLeader.dnodeId = pMember->info.dnodeId;
   strncpy(pGroup->assignedLeader.token, pMember->state.token, TSDB_ARB_TOKEN_SIZE);
+  pGroup->assignedLeader.acked = false;
 }
 
 static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) {
   pGroup->assignedLeader.dnodeId = 0;
   memset(pGroup->assignedLeader.token, 0, TSDB_ARB_TOKEN_SIZE);
+  pGroup->assignedLeader.acked = false;
 }
 
 static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
@@ -841,10 +848,10 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
     goto _OVER;
   }
 
-  mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s]",
+  mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]",
         pTrans->id, pNew->vgId, pNew->members[0].info.dnodeId, pNew->members[0].state.token,
         pNew->members[1].info.dnodeId, pNew->members[1].state.token, pNew->isSync, pNew->assignedLeader.dnodeId,
-        pNew->assignedLeader.token);
+        pNew->assignedLeader.token, pNew->assignedLeader.acked);
 
   mndTransAddArbGroupId(pTrans, pNew->vgId);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) {
@@ -1110,11 +1117,12 @@ bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char
     goto _OVER;
   }
 
-  if (pGroup->isSync) {
+  if (pGroup->assignedLeader.acked == false) {
     mndArbGroupDupObj(pGroup, pNewGroup);
     pNewGroup->isSync = false;
+    pNewGroup->assignedLeader.acked = true;
 
-    mInfo("vgId:%d, arb isSync is setting to false", vgId);
+    mInfo("vgId:%d, arb received assigned ack", vgId);
     updateAssigned = true;
     goto _OVER;
   }
@@ -1224,12 +1232,18 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       STR_WITH_MAXSIZE_TO_VARSTR(token, pGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)token, false);
+
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->assignedLeader.acked, false);
     } else {
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetNULL(pColInfo, numOfRows);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetNULL(pColInfo, numOfRows);
+
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetNULL(pColInfo, numOfRows);
     }
 
     taosThreadMutexUnlock(&pGroup->mutex);

From d815bb117be31fcd233a1c204fb3827b0a9dbd31 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 28 May 2024 16:56:23 +0800
Subject: [PATCH 049/132] add json for batch meta

---
 include/client/taos.h                   |   3 +-
 source/client/src/clientRawBlockWrite.c | 127 +++++++++++-------------
 utils/test/c/tmq_taosx_ci.c             |  20 +++-
 3 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/include/client/taos.h b/include/client/taos.h
index 66b5b92869..46e4e7633b 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -282,8 +282,7 @@ typedef enum tmq_res_t {
   TMQ_RES_INVALID = -1,
   TMQ_RES_DATA = 1,
   TMQ_RES_TABLE_META = 2,
-  TMQ_RES_METADATA = 3,
-  TMQ_RES_BATCH_TABLE_META = 4,
+  TMQ_RES_METADATA = 3
 } tmq_res_t;
 
 typedef struct tmq_topic_assignment {
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index d5f478d188..5e6d693562 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -26,11 +26,13 @@
 #define LOG_ID_TAG   "connId:0x%" PRIx64 ",reqId:0x%" PRIx64
 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId
 
+#define TMQ_META_VERSION "1.0"
+
 static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
 
 static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
 
-static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
+static cJSON* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
                                   int8_t t, SColCmprWrapper* pColCmprRow) {
   int8_t buildDefaultCompress = 0;
   if (pColCmprRow->nCols <= 0) {
@@ -124,9 +126,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
   }
   cJSON_AddItemToObject(json, "tags", tags);
 
-  string = cJSON_PrintUnformatted(json);
-  cJSON_Delete(json);
-  return string;
+  return json;
 }
 
 static int32_t setCompressOption(cJSON* json, uint32_t para) {
@@ -153,7 +153,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
   }
   return 0;
 }
-static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
+static cJSON* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
   SMAlterStbReq req = {0};
   cJSON*        json = NULL;
   char*         string = NULL;
@@ -247,18 +247,16 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
     default:
       break;
   }
-  string = cJSON_PrintUnformatted(json);
 
 end:
-  cJSON_Delete(json);
   tFreeSMAltertbReq(&req);
-  return string;
+  return json;
 }
 
-static char* processCreateStb(SMqMetaRsp* metaRsp) {
+static cJSON* processCreateStb(SMqMetaRsp* metaRsp) {
   SVCreateStbReq req = {0};
   SDecoder       coder;
-  char*          string = NULL;
+  cJSON*         pJson = NULL;
 
   uDebug("create stable data:%p", metaRsp);
   // decode and process req
@@ -269,17 +267,17 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) {
   if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
     goto _err;
   }
-  string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr);
+  pJson = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr);
 _err:
-  uDebug("create stable return, sql json:%s", string);
+  uDebug("create stable return, sql json:%s", cJSON_PrintUnformatted(pJson));
   tDecoderClear(&coder);
-  return string;
+  return pJson;
 }
 
-static char* processAlterStb(SMqMetaRsp* metaRsp) {
+static cJSON* processAlterStb(SMqMetaRsp* metaRsp) {
   SVCreateStbReq req = {0};
   SDecoder       coder;
-  char*          string = NULL;
+  cJSON*         pJson = NULL;
   uDebug("alter stable data:%p", metaRsp);
 
   // decode and process req
@@ -290,11 +288,11 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) {
   if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
     goto _err;
   }
-  string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen);
+  pJson = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen);
 _err:
-  uDebug("alter stable return, sql json:%s", string);
+  uDebug("alter stable return, sql json:%s", cJSON_PrintUnformatted(pJson));
   tDecoderClear(&coder);
-  return string;
+  return pJson;
 }
 
 static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
@@ -387,7 +385,7 @@ end:
   taosArrayDestroy(pTagVals);
 }
 
-static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) {
+static cJSON* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) {
   char*  string = NULL;
   cJSON* json = cJSON_CreateObject();
   if (json == NULL) {
@@ -412,16 +410,14 @@ static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) {
     cJSON_AddItemToArray(createList, create);
   }
   cJSON_AddItemToObject(json, "createList", createList);
-  string = cJSON_PrintUnformatted(json);
-  cJSON_Delete(json);
-  return string;
+  return json;
 }
 
-static char* processCreateTable(SMqMetaRsp* metaRsp) {
+static cJSON* processCreateTable(SMqMetaRsp* metaRsp) {
   SDecoder           decoder = {0};
   SVCreateTbBatchReq req = {0};
   SVCreateTbReq*     pCreateReq;
-  char*              string = NULL;
+  cJSON*             pJson = NULL;
   // decode
   uDebug("create table data:%p", metaRsp);
   void*   data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -435,15 +431,15 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) {
   if (req.nReqs > 0) {
     pCreateReq = req.pReqs;
     if (pCreateReq->type == TSDB_CHILD_TABLE) {
-      string = buildCreateCTableJson(req.pReqs, req.nReqs);
+      pJson = buildCreateCTableJson(req.pReqs, req.nReqs);
     } else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
-      string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid,
+      pJson = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid,
                                     TSDB_NORMAL_TABLE, &pCreateReq->colCmpr);
     }
   }
 
 _exit:
-  uDebug("create table return, sql json:%s", string);
+  uDebug("create table return, sql json:%s", cJSON_PrintUnformatted(pJson));
   for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
     pCreateReq = req.pReqs + iReq;
     taosMemoryFreeClear(pCreateReq->comment);
@@ -453,7 +449,7 @@ _exit:
     }
   }
   tDecoderClear(&decoder);
-  return string;
+  return pJson;
 }
 
 static char* processAutoCreateTable(STaosxRsp* rsp) {
@@ -482,7 +478,9 @@ static char* processAutoCreateTable(STaosxRsp* rsp) {
       goto _exit;
     }
   }
-  string = buildCreateCTableJson(pCreateReq, rsp->createTableNum);
+  cJSON* pJson = buildCreateCTableJson(pCreateReq, rsp->createTableNum);
+  string = cJSON_PrintUnformatted(pJson);
+  cJSON_Delete(pJson);
 _exit:
   uDebug("auto created table return, sql json:%s", string);
   for (int i = 0; i < rsp->createTableNum; i++) {
@@ -497,7 +495,7 @@ _exit:
   return string;
 }
 
-static char* processAlterTable(SMqMetaRsp* metaRsp) {
+static cJSON* processAlterTable(SMqMetaRsp* metaRsp) {
   SDecoder     decoder = {0};
   SVAlterTbReq vAlterTbReq = {0};
   char*        string = NULL;
@@ -622,19 +620,16 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
     default:
       break;
   }
-  string = cJSON_PrintUnformatted(json);
 
 _exit:
-  uDebug("alter table return, sql json:%s", string);
-  cJSON_Delete(json);
+  uDebug("alter table return, sql json:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&decoder);
-  return string;
+  return json;
 }
 
-static char* processDropSTable(SMqMetaRsp* metaRsp) {
+static cJSON* processDropSTable(SMqMetaRsp* metaRsp) {
   SDecoder     decoder = {0};
   SVDropStbReq req = {0};
-  char*        string = NULL;
   cJSON*       json = NULL;
   uDebug("processDropSTable data:%p", metaRsp);
 
@@ -659,18 +654,15 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
   cJSON* tableName = cJSON_CreateString(req.name);
   cJSON_AddItemToObject(json, "tableName", tableName);
 
-  string = cJSON_PrintUnformatted(json);
 _exit:
-  uDebug("processDropSTable return, sql json:%s", string);
-  cJSON_Delete(json);
+  uDebug("processDropSTable return, sql json:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&decoder);
-  return string;
+  return json;
 }
-static char* processDeleteTable(SMqMetaRsp* metaRsp) {
+static cJSON* processDeleteTable(SMqMetaRsp* metaRsp) {
   SDeleteRes req = {0};
   SDecoder   coder = {0};
   cJSON*     json = NULL;
-  char*      string = NULL;
 
   uDebug("processDeleteTable data:%p", metaRsp);
   // decode and process req
@@ -698,18 +690,15 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) {
   cJSON* sqlJson = cJSON_CreateString(sql);
   cJSON_AddItemToObject(json, "sql", sqlJson);
 
-  string = cJSON_PrintUnformatted(json);
 _exit:
-  uDebug("processDeleteTable return, sql json:%s", string);
-  cJSON_Delete(json);
+  uDebug("processDeleteTable return, sql json:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&coder);
-  return string;
+  return json;
 }
 
-static char* processDropTable(SMqMetaRsp* metaRsp) {
+static cJSON* processDropTable(SMqMetaRsp* metaRsp) {
   SDecoder         decoder = {0};
   SVDropTbBatchReq req = {0};
-  char*            string = NULL;
   cJSON*           json = NULL;
 
   uDebug("processDropTable data:%p", metaRsp);
@@ -743,12 +732,10 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
   }
   cJSON_AddItemToObject(json, "tableNameList", tableNameList);
 
-  string = cJSON_PrintUnformatted(json);
 _exit:
-  uDebug("processDropTable return, json sql:%s", string);
-  cJSON_Delete(json);
+  uDebug("processDropTable return, json sql:%s", cJSON_PrintUnformatted(json));
   tDecoderClear(&decoder);
-  return string;
+  return json;
 }
 
 static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
@@ -1805,6 +1792,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
     char  err[ERR_MSG_LEN] = {0};
     code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN);
     taosMemoryFree(fields);
+    taosMemoryFreeClear(pTableMeta);
     if (code != TSDB_CODE_SUCCESS) {
       SET_ERROR_MSG("table:%s, err:%s", tbName, err);
       goto end;
@@ -1997,6 +1985,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
     char  err[ERR_MSG_LEN] = {0};
     code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN);
     taosMemoryFree(fields);
+    taosMemoryFreeClear(pTableMeta);
     if (code != TSDB_CODE_SUCCESS) {
       SET_ERROR_MSG("table:%s, err:%s", tbName, err);
       goto end;
@@ -2027,7 +2016,7 @@ end:
   return code;
 }
 
-static char* processSimpleMeta(SMqMetaRsp* pMetaRsp) {
+static cJSON* processSimpleMeta(SMqMetaRsp* pMetaRsp) {
   if (pMetaRsp->resMsgType == TDMT_VND_CREATE_STB) {
     return processCreateStb(pMetaRsp);
   } else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_STB) {
@@ -2051,15 +2040,15 @@ static char* processSimpleMeta(SMqMetaRsp* pMetaRsp) {
 static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) {
   SDecoder        coder;
   SMqBatchMetaRsp rsp = {0};
-  SArray*         strArray = NULL;
   tDecoderInit(&coder, pMsgRsp->pMetaBuff, pMsgRsp->metaBuffLen);
   if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) {
     goto _end;
   }
 
-  int64_t fullSize = 0;
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddStringToObject(pJson, "tmq_meta_version", TMQ_META_VERSION);
+  cJSON* pMetaArr = cJSON_CreateArray();
   int32_t num = taosArrayGetSize(rsp.batchMetaReq);
-  strArray = taosArrayInit(num, POINTER_BYTES);
   for (int32_t i = 0; i < num; i++) {
     int32_t len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i);
     void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
@@ -2069,26 +2058,19 @@ static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) {
     if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) {
       goto _end;
     }
-    char* subStr = processSimpleMeta(&metaRsp);
+    cJSON* pItem = processSimpleMeta(&metaRsp);
     tDeleteMqMetaRsp(&metaRsp);
-    fullSize += strlen(subStr);
-    taosArrayPush(strArray, &subStr);
+    cJSON_AddItemToArray(pMetaArr, pItem);
   }
 
-  char* buf = (char*)taosMemoryCalloc(1, fullSize + num + 1);
-  for (int32_t i = 0; i < num; i++) {
-    char* subStr = taosArrayGetP(strArray, i);
-    strcat(buf, subStr);
-    if (i < num - 1) {
-      strcat(buf, "\n");
-    }
-  }
-  taosArrayDestroyP(strArray, taosMemoryFree);
+  cJSON_AddItemToObject(pJson, "metas", pMetaArr);
   tDeleteMqBatchMetaRsp(&rsp);
-  return buf;
+  char* fullStr = cJSON_PrintUnformatted(pJson);
+  cJSON_Delete(pJson);
+  return fullStr;
 
 _end:
-  taosArrayDestroyP(strArray, taosMemoryFree);
+  cJSON_Delete(pJson);
   tDeleteMqBatchMetaRsp(&rsp);
   return NULL;
 }
@@ -2109,7 +2091,10 @@ char* tmq_get_json_meta(TAOS_RES* res) {
   }
 
   SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
-  return processSimpleMeta(&pMetaRspObj->metaRsp);
+  cJSON* pJson = processSimpleMeta(&pMetaRspObj->metaRsp);
+  char* string = cJSON_PrintUnformatted(pJson);
+  cJSON_Delete(pJson);
+  return string;
 }
 
 void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index ef2d70f54f..c1b312335f 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -18,6 +18,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
+#include "cJSON.h"
 #include "taos.h"
 #include "tmsg.h"
 #include "types.h"
@@ -61,8 +62,23 @@ static void msg_process(TAOS_RES* msg) {
     if (result) {
       printf("meta result: %s\n", result);
       if (g_fp && strcmp(result, "") != 0) {
-        taosFprintfFile(g_fp, result);
-        taosFprintfFile(g_fp, "\n");
+        // RES_TYPE__TMQ_BATCH_META
+        if ((*(int8_t*)msg) == 5) {
+          cJSON* pJson = cJSON_Parse(result);
+          cJSON* pJsonArray = cJSON_GetObjectItem(pJson, "metas");
+          int32_t num = cJSON_GetArraySize(pJsonArray);
+          for (int32_t i = 0; i < num; i++) {
+            cJSON* pJsonItem = cJSON_GetArrayItem(pJsonArray, i);
+            char* itemStr = cJSON_PrintUnformatted(pJsonItem);
+            taosFprintfFile(g_fp, itemStr);
+            tmq_free_json_meta(itemStr);
+            taosFprintfFile(g_fp, "\n");
+          }
+          cJSON_Delete(pJson);
+        } else {
+          taosFprintfFile(g_fp, result);
+          taosFprintfFile(g_fp, "\n");
+        }
       }
     }
     tmq_free_json_meta(result);

From 8d54d45054f9e2f85e7a045e6a45a56750efc689 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 28 May 2024 17:30:14 +0800
Subject: [PATCH 050/132] fix(stream): fix error found by CI.

---
 source/dnode/vnode/src/tq/tq.c             |  4 ----
 source/dnode/vnode/src/tqCommon/tqCommon.c |  2 +-
 source/libs/stream/src/streamCheckpoint.c  | 18 +++++++++++++-----
 source/libs/stream/src/streamDispatch.c    |  9 +++++----
 4 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 037c2a7b7a..b54429f6b6 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1088,10 +1088,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   int32_t      len = pMsg->contLen - sizeof(SMsgHead);
   int32_t      code = 0;
 
-//  if (pTq->pStreamMeta->vgId == 2) {
-//    ASSERT(0);
-//  }
-
   // disable auto rsp to mnode
   pRsp->info.handle = NULL;
 
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index a137086d21..f87556e24e 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -864,7 +864,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
 }
 
 int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
-  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg;
+  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont;
 
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId);
   if (pTask == NULL) {
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 43b39b8574..968493b595 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -562,11 +562,11 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
 
   int32_t vgId = pTask->pMeta->vgId;
   int64_t now = taosGetTimestampMs();
-  stDebug("s-task:%s vgId:%d checkpoint-trigger monit start, ts:%" PRId64, pTask->id.idStr, vgId, now);
+  stDebug("s-task:%s vgId:%d checkpoint-trigger monitor start, ts:%" PRId64, pTask->id.idStr, vgId, now);
 
   taosThreadMutexLock(&pTask->lock);
   SStreamTaskState* pState = streamTaskGetStatus(pTask);
-  if (pState->state == TASK_STATUS__CK) {
+  if (pState->state != TASK_STATUS__CK) {
     stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger", pTask->id.idStr, vgId);
     taosThreadMutexUnlock(&pTask->lock);
     return;
@@ -599,12 +599,14 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
   }
 
   // do send retrieve checkpoint trigger msg to upstream
+  int32_t size = taosArrayGetSize(pNotSendList);
   doSendRetrieveTriggerMsg(pTask, pNotSendList);
   taosThreadMutexUnlock(&pActiveInfo->lock);
 
   // check every 100ms
-  if (taosArrayGetSize(pNotSendList) > 0) {
+  if (size > 0) {
     taosTmrReset(checkpointTriggerMonitorFn, 10000, pTask, streamTimer, &pActiveInfo->pCheckTmr);
+    stDebug("s-task:%s start monitor trigger in 10sec", pTask->id.idStr);
   }
 
   taosArrayDestroy(pNotSendList);
@@ -614,8 +616,11 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
   int32_t     code = 0;
   int32_t     vgId = pTask->pMeta->vgId;
   const char* pId = pTask->id.idStr;
+  int32_t     size = taosArrayGetSize(pNotSendList);
 
-  for (int32_t i = 0; i < taosArrayGetSize(pNotSendList); i++) {
+  stDebug("s-task:%s start to send trigger-retrieve msg to %d upstream(s)", pId, size);
+
+  for (int32_t i = 0; i < size; i++) {
     SStreamUpstreamEpInfo* pUpstreamTask = taosArrayGet(pNotSendList, i);
 
     SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq));
@@ -633,10 +638,13 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
     pReq->upstreamNodeId = pUpstreamTask->nodeId;
     pReq->checkpointId = pTask->chkInfo.pActiveInfo->activeId;
 
+
     SRpcMsg rpcMsg = {0};
-    initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE, pReq, sizeof(SRetrieveChkptTriggerReq));
+    initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq));
 
     code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg);
+    stDebug("s-task:%s vgId:%d send retrieve msg to 0x%x checkpointId:%" PRId64, pId, vgId, pUpstreamTask->taskId,
+            pReq->checkpointId);
   }
 
   return TSDB_CODE_SUCCESS;
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index c8a626a739..821ae68497 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -595,8 +595,9 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
     return 0;
   }
 
-  ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK || pBlock->type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
-         pBlock->type == STREAM_INPUT__TRANS_STATE);
+  int32_t type = pBlock->type;
+  ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
+         type == STREAM_INPUT__TRANS_STATE);
 
   pTask->execInfo.dispatch += 1;
   pTask->msgInfo.startTs = taosGetTimestampMs();
@@ -607,7 +608,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
   } else {  // todo handle build dispatch msg failed
   }
 
-  if (pBlock->type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
+  if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
     streamTaskInitTriggerDispatchInfo(pTask);
   }
 
@@ -829,7 +830,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
     }
   } else {
     taosArrayPush(pActiveInfo->pReadyMsgList, &info);
-    stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size);
+    stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1);
   }
 
   taosThreadMutexUnlock(&pActiveInfo->lock);

From a2255ca35558af614f6acdee7f9824ce5857c2f2 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 28 May 2024 18:34:37 +0800
Subject: [PATCH 051/132] add tmq param

---
 include/common/tmsg.h              |  5 +++++
 source/client/src/clientTmq.c      | 11 +++++++++++
 source/common/src/tmsg.c           | 16 ++++++++++++++++
 source/dnode/vnode/src/tq/tqUtil.c | 30 +++++++++++++++++++++++-------
 4 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 1657a17ff0..7917222542 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2744,6 +2744,7 @@ typedef struct {
   int32_t autoCommitInterval;
   int8_t  resetOffsetCfg;
   int8_t  enableReplay;
+  int8_t  enableBatchMeta;
 } SCMSubscribeReq;
 
 static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
@@ -2764,6 +2765,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
   tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval);
   tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg);
   tlen += taosEncodeFixedI8(buf, pReq->enableReplay);
+  tlen += taosEncodeFixedI8(buf, pReq->enableBatchMeta);
 
   return tlen;
 }
@@ -2788,6 +2790,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq
   buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval);
   buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg);
   buf = taosDecodeFixedI8(buf, &pReq->enableReplay);
+  buf = taosDecodeFixedI8(buf, &pReq->enableBatchMeta);
   return buf;
 }
 
@@ -2976,6 +2979,7 @@ typedef struct {
 
 int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
 int tDecodeSVCreateTbBatchReq(SDecoder* pCoder, SVCreateTbBatchReq* pReq);
+void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq* pReq);
 
 typedef struct {
   int32_t        code;
@@ -3893,6 +3897,7 @@ typedef struct {
   STqOffsetVal reqOffset;
   int8_t       enableReplay;
   int8_t       sourceExcluded;
+  int8_t       enableBatchMeta;
 } SMqPollReq;
 
 int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 69a4503680..8038273b43 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -71,6 +71,7 @@ struct tmq_conf_t {
   char*          pass;
   tmq_commit_cb* commitCb;
   void*          commitCbUserParam;
+  int8_t         enableBatchMeta;
 };
 
 struct tmq_t {
@@ -87,6 +88,7 @@ struct tmq_t {
   uint64_t       consumerId;
   tmq_commit_cb* commitCb;
   void*          commitCbUserParam;
+  int8_t         enableBatchMeta;
 
   // status
   SRWLatch lock;
@@ -269,6 +271,7 @@ tmq_conf_t* tmq_conf_new() {
   conf->autoCommit = true;
   conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL;
   conf->resetOffset = TMQ_OFFSET__RESET_LATEST;
+  conf->enableBatchMeta = false;
 
   return conf;
 }
@@ -398,6 +401,11 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     return TMQ_CONF_OK;
   }
 
+  if (strcasecmp(key, "msg.enable.batchmeta") == 0) {
+    conf->enableBatchMeta = (taosStr2int64(value) != 0) ? true : false;
+    return TMQ_CONF_OK;
+  }
+
   return TMQ_CONF_UNKNOWN;
 }
 
@@ -1122,6 +1130,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->resetOffsetCfg = conf->resetOffset;
   pTmq->replayEnable = conf->replayEnable;
   pTmq->sourceExcluded = conf->sourceExcluded;
+  pTmq->enableBatchMeta = conf->enableBatchMeta;
   if (conf->replayEnable) {
     pTmq->autoCommit = false;
   }
@@ -1199,6 +1208,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   req.autoCommitInterval = tmq->autoCommitInterval;
   req.resetOffsetCfg = tmq->resetOffsetCfg;
   req.enableReplay = tmq->replayEnable;
+  req.enableBatchMeta = tmq->enableBatchMeta;
 
   for (int32_t i = 0; i < sz; i++) {
     char* topic = taosArrayGetP(container, i);
@@ -1623,6 +1633,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
   pReq->reqId = generateRequestId();
   pReq->enableReplay = tmq->replayEnable;
   pReq->sourceExcluded = tmq->sourceExcluded;
+  pReq->enableBatchMeta = tmq->enableBatchMeta;
 }
 
 SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 3f76d03289..c17c522d4f 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -7199,6 +7199,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
     pHead->vgId = htonl(pReq->head.vgId);
     pHead->contLen = htonl(tlen + headLen);
   }
+  if (tEncodeI8(&encoder, pReq->enableBatchMeta) < 0) return -1;
 
   return tlen + headLen;
 }
@@ -7228,6 +7229,12 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
     if (tDecodeI8(&decoder, &pReq->sourceExcluded) < 0) return -1;
   }
 
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI8(&decoder, &pReq->enableBatchMeta) < 0) return -1;
+  } else {
+    pReq->enableBatchMeta = false;
+  }
+
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -8362,6 +8369,15 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) {
   return 0;
 }
 
+void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq* pReq) {
+  for (int32_t iReq = 0; iReq < pReq->nReqs; iReq++) {
+    SVCreateTbReq* pCreateReq = pReq->pReqs + iReq;
+    if (pCreateReq->type == TSDB_CHILD_TABLE) {
+      taosArrayDestroy(pCreateReq->ctb.tagName);
+    }
+  }
+}
+
 int tEncodeSVCreateTbRsp(SEncoder *pCoder, const SVCreateTbRsp *pRsp) {
   if (tStartEncode(pCoder) < 0) return -1;
 
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 4b1ff04331..496201e79e 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -174,7 +174,7 @@ end : {
 }
 }
 
-#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \
+#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \
   SDecoder           decoder = {0};\
   TYPE               req = {0}; \
   void*   data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \
@@ -184,11 +184,16 @@ end : {
     tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d",  \
             pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \
     fetchVer++; \
+    DELETE_FUNC(&req); \
     tDecoderClear(&decoder); \
     continue; \
   } \
+  DELETE_FUNC(&req); \
   tDecoderClear(&decoder);
 
+static void tDeleteCommon(void* parm) {
+}
+
 static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
                                                   SRpcMsg* pMsg, STqOffsetVal* offset) {
   int             code = 0;
@@ -266,17 +271,28 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
 
         if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) {
           if (pHead->msgType == TDMT_VND_CREATE_TABLE) {
-            PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq)
+            PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq, tDeleteSVCreateTbBatchReq)
           } else if (pHead->msgType == TDMT_VND_ALTER_TABLE) {
-            PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq)
+            PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq, tDeleteCommon)
           } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) {
-            PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq)
+            PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq, tDeleteCommon)
           } else if (pHead->msgType == TDMT_VND_DELETE) {
-            PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes)
+            PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes, tDeleteCommon)
           }
         }
 
-        tqDebug("fetch meta msg, ver:%" PRId64 ", vgId:%d, type:%s", pHead->version, vgId, TMSG_INFO(pHead->msgType));
+        tqDebug("fetch meta msg, ver:%" PRId64 ", vgId:%d, type:%s, enable batch meta:%d", pHead->version, vgId,
+                TMSG_INFO(pHead->msgType), pRequest->enableBatchMeta);
+        if (!pRequest->enableBatchMeta) {
+          SMqMetaRsp metaRsp = {0};
+          tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
+          metaRsp.resMsgType = pHead->msgType;
+          metaRsp.metaRspLen = pHead->bodyLen;
+          metaRsp.metaRsp = pHead->body;
+          code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId);
+          goto end;
+        }
+
         if (!btMetaRsp.batchMetaReq) {
           btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES);
           btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t));
@@ -348,7 +364,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
   }
 
 end:
-
+  tDeleteMqBatchMetaRsp(&btMetaRsp);
   tDeleteSTaosxRsp(&taosxRsp);
   return code;
 }

From 4bfddf5020e0532c12f90ab548f892a8dd83c6b7 Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Tue, 28 May 2024 19:09:56 +0800
Subject: [PATCH 052/132] fix: ci test case

---
 tests/system-test/0-others/information_schema.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py
index 944b2fbb1e..35f5e39214 100644
--- a/tests/system-test/0-others/information_schema.py
+++ b/tests/system-test/0-others/information_schema.py
@@ -222,7 +222,7 @@ class TDTestCase:
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
         tdLog.info(len(tdSql.queryResult))
-        tdSql.checkEqual(True, len(tdSql.queryResult) in range(254, 255))
+        tdSql.checkEqual(True, len(tdSql.queryResult) in range(255, 256))
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
         tdSql.checkEqual(54, len(tdSql.queryResult))
@@ -336,7 +336,7 @@ class TDTestCase:
             tdSql.checkEqual(True, result[i][1] in key_status_list[1])
             index += 1
         tdSql.checkEqual(True, index > 0)
-                    
+
         tdSql.query(f'show encryptions')
         result = tdSql.queryResult
         index = 0
@@ -344,7 +344,7 @@ class TDTestCase:
             tdSql.checkEqual(True, result[i][1] in key_status_list[1])
             index += 1
         tdSql.checkEqual(True, index > 0)
-        
+
         # loaded/sm4
         tdSql.execute('drop database if exists db2')
         tdSql.execute('create encrypt_key \'12345678\'')
@@ -357,7 +357,7 @@ class TDTestCase:
             tdSql.checkEqual(True, result[i][1] in key_status_list[3])
             index += 1
         tdSql.checkEqual(True, index > 0)
-                    
+
         tdSql.query(f'show encryptions')
         result = tdSql.queryResult
         index = 0

From 2cf657c76707bb5ca2920459cae3dd2fc81ab57b Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 28 May 2024 19:33:38 +0800
Subject: [PATCH 053/132] add ci

---
 utils/test/c/tmq_taosx_ci.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index c1b312335f..51d134a463 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -33,6 +33,7 @@ typedef struct {
   int  srcVgroups;
   int  dstVgroups;
   char dir[256];
+  bool btMeta;
 } Config;
 
 Config g_conf = {0};
@@ -600,6 +601,10 @@ tmq_t* build_consumer() {
     tmq_conf_set(conf, "experimental.snapshot.enable", "true");
   }
 
+  if (g_conf.btMeta) {
+    tmq_conf_set(conf, "msg.enable.batchmeta", "true");
+  }
+
   tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
   tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
   assert(tmq);
@@ -1184,6 +1189,8 @@ int main(int argc, char* argv[]) {
       g_conf.subTable = true;
     } else if (strcmp(argv[i], "-onlymeta") == 0) {
       g_conf.meta = 1;
+    } else if (strcmp(argv[i], "-bt") == 0) {
+      g_conf.btMeta = true;
     }
   }
 

From d1d868f23941d669bfd5b9563709c77f4a97c3d2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 00:08:29 +0800
Subject: [PATCH 054/132] refactor: do some internal refactor and set the
 pActiveInfo for rsma.

---
 include/dnode/vnode/tqCommon.h             |  1 +
 include/libs/stream/streammsg.h            |  8 +++
 include/libs/stream/tstream.h              |  5 +-
 source/dnode/snode/src/snode.c             |  4 ++
 source/dnode/vnode/src/inc/vnodeInt.h      |  3 +-
 source/dnode/vnode/src/sma/smaRollup.c     |  2 +-
 source/dnode/vnode/src/tq/tq.c             |  6 +-
 source/dnode/vnode/src/tq/tqStreamTask.c   |  3 +-
 source/dnode/vnode/src/tqCommon/tqCommon.c | 25 ++++++-
 source/dnode/vnode/src/vnd/vnodeSvr.c      |  4 +-
 source/libs/stream/src/streamCheckStatus.c |  2 +-
 source/libs/stream/src/streamCheckpoint.c  | 77 +++++++++++++++-------
 source/libs/stream/src/streamDispatch.c    |  2 +-
 source/libs/stream/src/streamTask.c        |  1 -
 14 files changed, 104 insertions(+), 39 deletions(-)

diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h
index 6d52b10182..0076d79312 100644
--- a/include/dnode/vnode/tqCommon.h
+++ b/include/dnode/vnode/tqCommon.h
@@ -37,6 +37,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta);
 int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta);
 int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
 int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
 int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
diff --git a/include/libs/stream/streammsg.h b/include/libs/stream/streammsg.h
index 87c756b10c..96701fe21d 100644
--- a/include/libs/stream/streammsg.h
+++ b/include/libs/stream/streammsg.h
@@ -181,6 +181,14 @@ typedef struct SRetrieveChkptTriggerReq {
   int64_t  downstreamTaskId;
 } SRetrieveChkptTriggerReq;
 
+typedef struct SCheckpointTriggerRsp {
+  int64_t  streamId;
+  int64_t  checkpointId;
+  int32_t  upstreamTaskId;
+  int32_t  taskId;
+  int32_t  transId;
+} SCheckpointTriggerRsp;
+
 typedef struct {
   SMsgHead head;
   int64_t  streamId;
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 185ab7ad51..0d01913235 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -678,8 +678,7 @@ bool    streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeI
 void    streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal);
 void    streamTaskInitTriggerDispatchInfo(SStreamTask* pTask);
 void    streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId);
-int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t checkpointType, int32_t dstTaskId, int32_t vgId,
-                                           SEpSet* pEpset);
+int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pRpcInfo);
 
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
 
@@ -753,6 +752,7 @@ tmr_h   streamTimerGetInstance();
 
 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
+int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
 int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
@@ -764,6 +764,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
 int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId,
                                               SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
 int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq);
+SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo();
 
 // stream task state machine, and event handling
 SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index ac10aa83a4..89ab6d52c3 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -136,6 +136,10 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
       return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg);
     case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
       return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg);
+    case TDMT_STREAM_RETRIEVE_TRIGGER:
+      return tqStreamTaskProcessRetrieveTriggerReq(pSnode->pMeta, pMsg);
+    case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:
+      return tqStreamTaskProcessRetrieveTriggerRsp(pSnode->pMeta, pMsg);
     default:
       sndError("invalid snode msg:%d", pMsg->msgType);
       ASSERT(0);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 59599fdae6..b369bd6039 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -254,7 +254,8 @@ int     tqScanWalAsync(STQ* pTq, bool ckPause);
 int32_t tqStopStreamTasksAsync(STQ* pTq);
 int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp);
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
-int32_t tqProcessTaskRetrieveTriggerMsg(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskRetrieveTriggerRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg);
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 6babcf3c80..7138ecbeaa 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -298,7 +298,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
     pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta);
     tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
     pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask);
-
+    pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo();
     pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1);
     if (!pStreamState) {
       terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index b54429f6b6..d8460d2cb5 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1240,10 +1240,14 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
   return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg);
 }
 
-int32_t tqProcessTaskRetrieveTriggerMsg(STQ* pTq, SRpcMsg* pMsg) {
+int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg) {
   return tqStreamTaskProcessRetrieveTriggerReq(pTq->pStreamMeta, pMsg);
 }
 
+int32_t tqProcessTaskRetrieveTriggerRsp(STQ* pTq, SRpcMsg* pMsg) {
+  return tqStreamTaskProcessRetrieveTriggerRsp(pTq->pStreamMeta, pMsg);
+}
+
 // this function is needed, do not try to remove it.
 int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) {
   return tqStreamProcessStreamHbRsp(pTq->pStreamMeta, pMsg);
diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c
index 5a29f67ae3..bcf17bf1e1 100644
--- a/source/dnode/vnode/src/tq/tqStreamTask.c
+++ b/source/dnode/vnode/src/tq/tqStreamTask.c
@@ -291,8 +291,7 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems
         }
       } else {
         walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
-        tqError("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id,
-                pTask->chkInfo.nextProcessVer);
+        tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer);
         break;
       }
     }
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index f87556e24e..ee3f1a3760 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -887,9 +887,9 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
 
     if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) {
       // re-send the lost checkpoint-trigger msg to downstream task
-      SEpSet* pEpset = streamTaskGetDownstreamEpInfo(pTask, pReq->downstreamTaskId);
-      streamTaskSendCheckpointTriggerMsg(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pReq->downstreamTaskId,
-                                         pReq->downstreamNodeId, pEpset);
+      tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
+              (int32_t)pReq->downstreamTaskId, checkpointId, transId);
+      streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info);
     } else { // not send checkpoint-trigger yet, wait
       int32_t recv = 0, total = 0;
       streamTaskGetTriggerRecvStatus(pTask, &recv, &total);
@@ -914,6 +914,25 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
+  SCheckpointTriggerRsp* pRsp = pMsg->pCont;
+
+  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId);
+  if (pTask == NULL) {
+    tqError(
+        "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
+        pMeta->vgId, pRsp->taskId);
+    return TSDB_CODE_STREAM_TASK_NOT_EXIST;
+  }
+
+  tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%"PRId64", transId:%d",
+      pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
+
+  streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
+  streamMetaReleaseTask(pMeta, pTask);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
   SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 490c3f08ce..426f85fa5e 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -843,7 +843,7 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
     case TDMT_STREAM_TASK_CHECKPOINT_READY:
       return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
     case TDMT_STREAM_RETRIEVE_TRIGGER:
-      return tqProcessTaskRetrieveTriggerMsg(pVnode->pTq, pMsg);
+      return tqProcessTaskRetrieveTriggerReq(pVnode->pTq, pMsg);
     case TDMT_MND_STREAM_HEARTBEAT_RSP:
       return tqProcessStreamHbRsp(pVnode->pTq, pMsg);
     case TDMT_MND_STREAM_REQ_CHKPT_RSP:
@@ -851,7 +851,7 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
     case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
       return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
     case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:
-      return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
+      return tqProcessTaskRetrieveTriggerRsp(pVnode->pTq, pMsg);
     case TDMT_VND_GET_STREAM_PROGRESS:
       return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
     default:
diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c
index f083ff8a61..d5c676433f 100644
--- a/source/libs/stream/src/streamCheckStatus.c
+++ b/source/libs/stream/src/streamCheckStatus.c
@@ -246,7 +246,7 @@ int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp*
 }
 
 int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTaskCheckRsp* pRsp,
-                           SRpcHandleInfo* pRpcInfo, int32_t taskId) {
+                               SRpcHandleInfo* pRpcInfo, int32_t taskId) {
   SEncoder encoder;
   int32_t  code;
   int32_t  len;
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 968493b595..fb3c705a4d 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -118,23 +118,40 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo
   return appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER);
 }
 
-int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t checkpointType, int32_t dstTaskId, int32_t vgId,
-                                           SEpSet* pEpset) {
-  SStreamDataBlock* pChkpoint = createChkptTriggerBlock(pTask, checkpointType);
+int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp) {
+  ASSERT(pTask->info.taskLevel != TASK_LEVEL__SOURCE);
 
-  pChkpoint->srcTaskId = pTask->id.taskId;
-  pChkpoint->srcVgId = pTask->pMeta->vgId;
-
-  int32_t code = streamTaskBuildAndSendTriggerMsg(pTask, pChkpoint, dstTaskId, vgId, pEpset);
-  if (code == TSDB_CODE_SUCCESS) {
-    stDebug("s-task:%s build and send checkpoint-trigger dispatch msg succ, stage:%" PRId64, pTask->id.idStr,
-            pTask->pMeta->stage);
-  } else {
-    // todo handle send data failure
-    stError("s-task:%s failed to build and send trigger msg", pTask->id.idStr);
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+  if (pInfo->transId != pRsp->transId || pInfo->activeId != pRsp->checkpointId) {
+    // todo handle error
+    return -1;
   }
 
-  return code;
+  taosThreadMutexLock(&pTask->lock);
+  SStreamTaskState* pState = streamTaskGetStatus(pTask);
+  if (pState->state != TASK_STATUS__CK) {
+    // todo handle error
+    taosThreadMutexUnlock(&pTask->lock);
+    return -1;
+  }
+
+  taosThreadMutexUnlock(&pTask->lock);
+
+  appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pRpcInfo) {
+  SCheckpointTriggerRsp* pRsp = rpcMallocCont(sizeof(SCheckpointTriggerRsp));
+  pRsp->streamId = pTask->id.streamId;
+  pRsp->upstreamTaskId = pTask->id.taskId;
+  pRsp->taskId = dstTaskId;
+  pRsp->checkpointId = pTask->chkInfo.pActiveInfo->activeId;
+  pRsp->transId = pTask->chkInfo.pActiveInfo->transId;
+
+  SRpcMsg rspMsg = {.code = 0, .pCont = pRsp, .contLen = sizeof(SCheckpointTriggerRsp), .info = *pRpcInfo};
+  tmsgSendRsp(&rspMsg);
+  return 0;
 }
 
 int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) {
@@ -617,8 +634,15 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
   int32_t     vgId = pTask->pMeta->vgId;
   const char* pId = pTask->id.idStr;
   int32_t     size = taosArrayGetSize(pNotSendList);
+  int32_t     numOfUpstream = streamTaskGetNumOfUpstream(pTask);
 
-  stDebug("s-task:%s start to send trigger-retrieve msg to %d upstream(s)", pId, size);
+  if (size <= 0) {
+    stDebug("s-task:%s all upstream checkpoint trigger recved, no need to send retrieve", pId);
+    return code;
+  }
+
+  stDebug("s-task:%s %d/%d not recv checkpoint-trigger from upstream(s), start to send trigger-retrieve", pId, size,
+          numOfUpstream);
 
   for (int32_t i = 0; i < size; i++) {
     SStreamUpstreamEpInfo* pUpstreamTask = taosArrayGet(pNotSendList, i);
@@ -643,21 +667,23 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
     initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq));
 
     code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg);
-    stDebug("s-task:%s vgId:%d send retrieve msg to 0x%x checkpointId:%" PRId64, pId, vgId, pUpstreamTask->taskId,
-            pReq->checkpointId);
+    stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId, vgId,
+            pUpstreamTask->taskId, pUpstreamTask->nodeId, pReq->checkpointId);
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
 bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) {
-  SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
+  int64_t                now = taosGetTimestampMs();
+  const char*            id = pTask->id.idStr;
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+  SStreamTaskState*      pStatus = streamTaskGetStatus(pTask);
+
   if (pStatus->state != TASK_STATUS__CK) {
     return false;
   }
 
-  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
-
   taosThreadMutexLock(&pInfo->lock);
   if (!pInfo->dispatchTrigger) {
     taosThreadMutexUnlock(&pInfo->lock);
@@ -671,12 +697,15 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
     }
 
     // has send trigger msg to downstream node,
+    double before = (now - pSendInfo->sendTs) / 1000.0;
     if (pSendInfo->recved) {
-      stWarn("s-task:%s checkpoint-trigger msg send at:%"PRId64" and recv confirmed, checkpointId:%"PRId64 ", transId:%d",
-          pTask->id.idStr, pSendInfo->sendTs, pInfo->activeId, pInfo->transId);
+      stWarn("s-task:%s checkpoint-trigger msg already send at:%" PRId64
+             "(%.2fs before) and recv confirmed by downstream:0x%x, checkpointId:%" PRId64 ", transId:%d",
+             id, pSendInfo->sendTs, before, pSendInfo->taskId, pInfo->activeId, pInfo->transId);
     } else {
-      stWarn("s-task:%s checkpoint-trigger send at:%"PRId64", checkpointId:%"PRId64", transId:%d", pTask->id.idStr,
-          pSendInfo->sendTs, pInfo->activeId, pInfo->transId);
+      stWarn("s-task:%s checkpoint-trigger already send at:%" PRId64 "(%.2fs before), checkpointId:%" PRId64
+             ", transId:%d",
+             id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId);
     }
 
     taosThreadMutexUnlock(&pInfo->lock);
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 821ae68497..ecea494be2 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -480,7 +480,7 @@ static void doRetryDispatchData(void* param, void* tmrId) {
 void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration) {
   pTask->msgInfo.retryCount++;
 
-  stWarn("s-task:%s retry send dispatch data in %" PRId64 "ms, in timer msgId:%d, retryTimes:%d", pTask->id.idStr,
+  stTrace("s-task:%s retry send dispatch data in %" PRId64 "ms, in timer msgId:%d, retryTimes:%d", pTask->id.idStr,
          waitDuration, pTask->execInfo.dispatch, pTask->msgInfo.retryCount);
 
   if (pTask->msgInfo.pTimer != NULL) {
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index e0415c8467..6d30cc6759 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -25,7 +25,6 @@ static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo);
 static void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated);
 static void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdate);
 static void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo);
-static SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo();
 
 static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) {
   int32_t childId = taosArrayGetSize(pArray);

From 225dfb1a9335d2d6cc3ed03d03b87985aede07a1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 00:30:42 +0800
Subject: [PATCH 055/132] enh(stream): check checkpoint-trigger msg every
 100ms.

---
 source/libs/stream/inc/streamInt.h         |  4 ++-
 source/libs/stream/src/streamCheckStatus.c |  2 +-
 source/libs/stream/src/streamCheckpoint.c  | 39 +++++++++++++++++-----
 source/libs/stream/src/streamExec.c        |  4 +++
 4 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 68c3ab1a6b..10db53ea38 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -61,7 +61,9 @@ struct SActiveCheckpointInfo {
   bool          dispatchTrigger;
   SArray*       pDispatchTriggerList;   // SArray<STaskTriggerSendInfo>
   SArray*       pReadyMsgList;   // SArray<SStreamChkptReadyInfo*>
-  tmr_h        pCheckTmr;
+
+  int32_t       checkCounter;
+  tmr_h         pCheckTmr;
 };
 
 typedef struct {
diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c
index d5c676433f..11fecf7683 100644
--- a/source/libs/stream/src/streamCheckStatus.c
+++ b/source/libs/stream/src/streamCheckStatus.c
@@ -285,7 +285,7 @@ int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
   streamTaskInitTaskCheckInfo(pInfo, &pTask->outputInfo, taosGetTimestampMs());
 
   int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
-  stDebug("s-task:%s start check-rsp monit, ref:%d ", pTask->id.idStr, ref);
+  stDebug("s-task:%s start check-rsp monitor, ref:%d ", pTask->id.idStr, ref);
 
   if (pInfo->checkRspTmr == NULL) {
     pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer);
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index fb3c705a4d..414c4e2a76 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -36,8 +36,9 @@ static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, con
 static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask);
 static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType);
 static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList);
+static void    checkpointTriggerMonitorFn(void* param, void* tmrId);
+
 static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType);
-static void checkpointTriggerMonitorFn(void* param, void* tmrId);
 
 bool streamTaskIsAllUpstreamSendTrigger(SStreamTask* pTask) {
   SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
@@ -194,11 +195,13 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       return code;
     }
 
+    int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s start check-rsp monitor in 10s, ref:%d ", pTask->id.idStr, ref);
     SActiveCheckpointInfo* pActive = pTask->chkInfo.pActiveInfo;
     if (pActive->pCheckTmr == NULL) {
-      pActive->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 10000, pTask, streamTimer);
+      pActive->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
     } else {
-      taosTmrReset(checkpointTriggerMonitorFn, 10000, pTask, streamTimer, &pActive->pCheckTmr);
+      taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActive->pCheckTmr);
     }
   }
 
@@ -575,17 +578,33 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
 
 void checkpointTriggerMonitorFn(void* param, void* tmrId) {
   SStreamTask* pTask = param;
+  int32_t      vgId = pTask->pMeta->vgId;
+  int64_t      now = taosGetTimestampMs();
+  const char*  id = pTask->id.idStr;
+
   SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
 
-  int32_t vgId = pTask->pMeta->vgId;
-  int64_t now = taosGetTimestampMs();
-  stDebug("s-task:%s vgId:%d checkpoint-trigger monitor start, ts:%" PRId64, pTask->id.idStr, vgId, now);
+  // check the status every 100ms
+  if (streamTaskShouldStop(pTask)) {
+    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref);
+    streamMetaReleaseTask(pTask->pMeta, pTask);
+    return;
+  }
+
+  if (++pActiveInfo->checkCounter < 100) {
+    return;
+  }
+
+  pActiveInfo->checkCounter = 0;
+  stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, pTask->id.idStr, vgId, now);
 
   taosThreadMutexLock(&pTask->lock);
   SStreamTaskState* pState = streamTaskGetStatus(pTask);
   if (pState->state != TASK_STATUS__CK) {
     stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger", pTask->id.idStr, vgId);
     taosThreadMutexUnlock(&pTask->lock);
+    streamMetaReleaseTask(pTask->pMeta, pTask);
     return;
   }
   taosThreadMutexUnlock(&pTask->lock);
@@ -622,8 +641,12 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
 
   // check every 100ms
   if (size > 0) {
-    taosTmrReset(checkpointTriggerMonitorFn, 10000, pTask, streamTimer, &pActiveInfo->pCheckTmr);
-    stDebug("s-task:%s start monitor trigger in 10sec", pTask->id.idStr);
+    stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id);
+    taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pCheckTmr);
+  } else {
+    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref);
+    streamMetaReleaseTask(pTask->pMeta, pTask);
   }
 
   taosArrayDestroy(pNotSendList);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 87f239f31c..1e85b959a8 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -597,6 +597,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
     // dispatch checkpoint msg to all downstream tasks
     int32_t type = pInput->type;
     if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
+      if (pTask->pMeta->vgId == 2) {
+        taosSsleep(20);
+      }
+
       streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
       continue;
     }

From a9abe78eab4fc622d34aa005913043fade07258e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 09:16:37 +0800
Subject: [PATCH 056/132] other: comment invalid sleep for test purpose

---
 source/libs/stream/src/streamCheckpoint.c | 3 ++-
 source/libs/stream/src/streamExec.c       | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 414c4e2a76..756183a624 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -196,9 +196,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     }
 
     int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
-    stDebug("s-task:%s start check-rsp monitor in 10s, ref:%d ", pTask->id.idStr, ref);
+    stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
     SActiveCheckpointInfo* pActive = pTask->chkInfo.pActiveInfo;
     if (pActive->pCheckTmr == NULL) {
+      streamMetaAcquireOneTask(pTask);
       pActive->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
     } else {
       taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActive->pCheckTmr);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 1e85b959a8..2b92696b88 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -598,7 +598,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
     int32_t type = pInput->type;
     if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
       if (pTask->pMeta->vgId == 2) {
-        taosSsleep(20);
+//        taosSsleep(20);
       }
 
       streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);

From 7182e0948eb67a4d745a99bd270cff4cc6135aa9 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 29 May 2024 10:49:36 +0800
Subject: [PATCH 057/132] fix:[TD-30306]error in converity scan

---
 source/client/src/clientSml.c       |  2 +-
 source/client/src/clientSmlTelnet.c |  2 +-
 source/client/src/clientTmq.c       | 15 ++++++---------
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 6225cf703c..6eb7abe0eb 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -939,7 +939,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns,
   pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions));
   for (int32_t i = 0; i < pReq.numOfColumns; ++i) {
     SField           *pField = taosArrayGet(pColumns, i);
-    SFieldWithOptions fieldWithOption;
+    SFieldWithOptions fieldWithOption = {0};
     setFieldWithOptions(&fieldWithOption, pField);
     setDefaultOptionsForField(&fieldWithOption);
     taosArrayPush(pReq.pColumns, &fieldWithOption);
diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c
index f715f32556..bc0e560178 100644
--- a/source/client/src/clientSmlTelnet.c
+++ b/source/client/src/clientSmlTelnet.c
@@ -233,7 +233,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
 
   SSmlKv kvTs = {0};
   smlBuildTsKv(&kvTs, ts);
-  if (needConverTime) {
+  if (needConverTime && info->currSTableMeta != NULL) {
     kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
   }
 
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 3141e21f26..0bc4214c09 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1344,6 +1344,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     goto FAIL;
   }
 
+  if(pMsg->pData == NULL){
+    tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId);
+    code = TSDB_CODE_TSC_INTERNAL_ERROR;
+    goto FAIL;
+  }
+
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0);
   if (pRspWrapper == NULL) {
     tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId);
@@ -1356,11 +1362,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     goto END;
   }
 
-  if(pMsg->pData == NULL){
-    tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId);
-    goto FAIL;
-  }
-
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t clientEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < clientEpoch) {
@@ -2809,8 +2810,6 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep
   int64_t transporterId = 0;
   code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, sendInfo);
   if (code != 0) {
-    taosMemoryFree(buf);
-    taosMemoryFree(sendInfo);
     tsem_destroy(&pParam->sem);
     taosMemoryFree(pParam);
     return code;
@@ -3235,8 +3234,6 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
   int64_t transporterId = 0;
   code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
   if (code != 0) {
-    taosMemoryFree(msg);
-    taosMemoryFree(sendInfo);
     tsem_destroy(&pParam->sem);
     taosMemoryFree(pParam);
     return code;

From 5283ca87edacbd0800d346ee01fae597be14a335 Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Wed, 29 May 2024 03:08:15 +0000
Subject: [PATCH 058/132] fix/TD-30307

---
 source/libs/monitorfw/inc/taos_linked_list_i.h |  2 +-
 source/libs/monitorfw/src/taos_collector.c     | 10 +++++++---
 .../monitorfw/src/taos_collector_registry.c    |  4 ++++
 source/libs/monitorfw/src/taos_linked_list.c   |  2 ++
 source/libs/monitorfw/src/taos_map.c           |  4 ++--
 source/libs/monitorfw/src/taos_metric.c        | 18 ++++++++++++------
 6 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h
index ed11a76427..015f8a57ad 100644
--- a/source/libs/monitorfw/inc/taos_linked_list_i.h
+++ b/source/libs/monitorfw/inc/taos_linked_list_i.h
@@ -47,7 +47,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item);
 /**
  * @brief API PRIVATE Pop the first item off of the list
  */
-void *taos_linked_list_pop(taos_linked_list_t *self);
+//void *taos_linked_list_pop(taos_linked_list_t *self);
 
 /**
  * @brief API PRIVATE Returns the item at the head of the list or NULL if not present
diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c
index 17d324462c..414c02121d 100644
--- a/source/libs/monitorfw/src/taos_collector.c
+++ b/source/libs/monitorfw/src/taos_collector.c
@@ -33,6 +33,8 @@ taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self
 taos_collector_t *taos_collector_new(const char *name) {
   int r = 0;
   taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t));
+  if (self == NULL) return NULL;
+  memset(self, 0, sizeof(taos_collector_t));
   self->name = taos_strdup(name);
   self->metrics = taos_map_new();
   if (self->metrics == NULL) {
@@ -66,9 +68,11 @@ int taos_collector_destroy(taos_collector_t *self) {
   if (r) ret = r;
   self->metrics = NULL;
 
-  r = taos_string_builder_destroy(self->string_builder);
-  if (r) ret = r;
-  self->string_builder = NULL;
+  if(self->string_builder != NULL){
+    r = taos_string_builder_destroy(self->string_builder);
+    if (r) ret = r;
+    self->string_builder = NULL;
+  }
 
   taos_free((char *)self->name);
   self->name = NULL;
diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c
index c3ed0112c5..d4838ef301 100644
--- a/source/libs/monitorfw/src/taos_collector_registry.c
+++ b/source/libs/monitorfw/src/taos_collector_registry.c
@@ -59,6 +59,7 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) {
   r = pthread_rwlock_init(self->lock, NULL);
   if (r) {
     TAOS_LOG("failed to initialize rwlock");
+    taos_free(self);
     return NULL;
   }
   return self;
@@ -301,6 +302,9 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self,
 
 _OVER:
   tjsonDelete(pJson);
+  if(tmp_builder != NULL){
+    taos_string_builder_destroy(tmp_builder);
+  }
 
   return NULL;
 }
diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c
index 675400a6fa..2becb08a07 100644
--- a/source/libs/monitorfw/src/taos_linked_list.c
+++ b/source/libs/monitorfw/src/taos_linked_list.c
@@ -117,6 +117,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item) {
   return 0;
 }
 
+/*
 void *taos_linked_list_pop(taos_linked_list_t *self) {
   TAOS_ASSERT(self != NULL);
   if (self == NULL) return NULL;
@@ -141,6 +142,7 @@ void *taos_linked_list_pop(taos_linked_list_t *self) {
   }
   return item;
 }
+*/
 
 int taos_linked_list_remove(taos_linked_list_t *self, void *item) {
   TAOS_ASSERT(self != NULL);
diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c
index 8f0b83884e..ffb7d000fc 100644
--- a/source/libs/monitorfw/src/taos_map.c
+++ b/source/libs/monitorfw/src/taos_map.c
@@ -90,7 +90,7 @@ taos_map_t *taos_map_new() {
     return NULL;
   }
 
-  self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size);
+  self->addrs = taos_malloc(sizeof(taos_linked_list_t*) * self->max_size);
   self->free_value_fn = destroy_map_node_value_no_op;
 
   for (int i = 0; i < self->max_size; i++) {
@@ -273,7 +273,7 @@ int taos_map_ensure_space(taos_map_t *self) {
   if (r) return r;
 
   // Create a new array of addrs
-  taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max);
+  taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t*) * new_max);
 
   // Initialize the new array
   for (int i = 0; i < new_max; i++) {
diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c
index 4e9af35f34..5cecbc927f 100644
--- a/source/libs/monitorfw/src/taos_metric.c
+++ b/source/libs/monitorfw/src/taos_metric.c
@@ -33,6 +33,8 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name,
                                size_t label_key_count, const char **label_keys) {
   int r = 0;
   taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t));
+  if (self == NULL) return NULL;
+  memset(self, 0, sizeof(taos_metric_t));
   self->type = metric_type;
   int len = strlen(name) + 1;
   self->name = taos_malloc(len);
@@ -79,6 +81,7 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name,
   r = pthread_rwlock_init(self->rwlock, NULL);
   if (r) {
     TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR);
+    taos_free(self);
     return NULL;
   }
   return self;
@@ -91,9 +94,11 @@ int taos_metric_destroy(taos_metric_t *self) {
   int r = 0;
   int ret = 0;
 
-  r = taos_map_destroy(self->samples);
-  self->samples = NULL;
-  if (r) ret = r;
+  if(self->samples != NULL){
+    r = taos_map_destroy(self->samples);
+    self->samples = NULL;
+    if (r) ret = r;
+  }
 
   r = taos_metric_formatter_destroy(self->formatter);
   self->formatter = NULL;
@@ -152,8 +157,7 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const
   return NULL;
 
   // Get l_value
-  r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys,
-                                         label_values);
+  r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, label_values);
   if (r) {
     TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
   }
@@ -170,10 +174,12 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const
     sample = taos_metric_sample_new(self->type, l_value, 0.0);
     r = taos_map_set(self->samples, l_value, sample);
     if (r) {
+      taos_free((void *)l_value);
       TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
     }
   }
-  pthread_rwlock_unlock(self->rwlock);
+  r = pthread_rwlock_unlock(self->rwlock);
+  if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); 
   taos_free((void *)l_value);
   return sample;
 }

From 09efbad11741d8f5385602b6540df1c17d3278fd Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 11:16:26 +0800
Subject: [PATCH 059/132] fix(stream): drop related fill history task.

---
 source/libs/stream/src/streamCheckpoint.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 756183a624..a177c56f77 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -305,6 +305,17 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin
             " no need to update the checkpoint info, updated checkpointId:%" PRId64 " checkpointVer:%" PRId64 " ignored",
             id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer);
     taosThreadMutexUnlock(&pTask->lock);
+
+    { // destroy the related fill-history tasks
+      // drop task should not in the meta-lock, and drop the related fill-history task now
+      if (pReq->dropRelHTask) {
+        streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
+        int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
+        stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d",
+                id, vgId, pReq->taskId, numOfTasks);
+      }
+    }
+
     return TSDB_CODE_SUCCESS;
   }
 

From bb83ed2469501ad1554b2a40494edc4eedff3922 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 29 May 2024 11:33:17 +0800
Subject: [PATCH 060/132] fix: compact and commit concurrency problem

---
 source/dnode/vnode/src/tsdb/tsdbCommit2.c | 89 ++++++++++++++---------
 1 file changed, 55 insertions(+), 34 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index 1a08f2dc82..855bc12ff3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -560,20 +560,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
   code = tsdbCommitInfoInit(tsdb);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  taosThreadMutexLock(&tsdb->mutex);
-  TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
-    SFileSetCommitInfo info = {
-        .fid = fset->fid,
-        .hasDataToCommit = false,
-        .fset = fset,
-    };
-    if ((code = tsdbCommitInfoAdd(tsdb, &info))) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-  }
-  taosThreadMutexUnlock(&tsdb->mutex);
-
+  // scan time-series data
   iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
   for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
     STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
@@ -608,24 +595,68 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
 
       from.key.ts = maxKey + 1;
     }
+  }
 
-    // scan tomb data
-    for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) {
-      for (int32_t i = taosArrayGetSize(tsdb->commitInfo->arr) - 1; i >= 0; i--) {
-        int64_t             minKey, maxKey;
-        SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+  taosThreadMutexLock(&tsdb->mutex);
 
-        tsdbFidKeyRange(info->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
+  // copy existing file set
+  for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
+    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+    tsdbFSGetFSet(tsdb->pFS, info->fid, &fset);
+    if (fset) {
+      tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
+    }
+  }
 
-        if (pDelData->sKey > maxKey || pDelData->eKey < minKey) {
-          continue;
-        } else if (!info->hasDataToCommit) {
-          info->hasDataToCommit = true;
+  // scan tomb data
+  if (tsdb->imem->nDel > 0) {
+    TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
+      SFileSetCommitInfo *info;
+      SFileSetCommitInfo  tinfo = {
+           .fid = fset->fid,
+      };
+
+      // check if the file set already on the commit list
+      vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
+      if (info != NULL) {
+        continue;
+      }
+
+      int64_t minKey, maxKey;
+      tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
+      tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
+      for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
+        STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
+        for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) {
+          if (pDelData->sKey > maxKey || pDelData->eKey < minKey) {
+            continue;
+          } else {
+            tinfo.fid = fset->fid;
+            tinfo.hasDataToCommit = true;
+            tinfo.fset = fset;
+            if ((code = tsdbCommitInfoAdd(tsdb, &tinfo))) {
+              taosThreadMutexUnlock(&tsdb->mutex);
+              TSDB_CHECK_CODE(code, lino, _exit);
+            }
+            break;
+          }
+        }
+
+        if (tinfo.hasDataToCommit) {
+          break;
         }
       }
     }
   }
 
+  // begin tasks on file set
+  for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
+    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+    tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
+  }
+
+  taosThreadMutexUnlock(&tsdb->mutex);
+
 _exit:
   if (code) {
     tsdbCommitInfoDestroy(tsdb);
@@ -653,16 +684,6 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
   code = tsdbCommitInfoBuild(tsdb);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  STFileSet *fset;
-  taosThreadMutexLock(&tsdb->mutex);
-  for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
-    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
-    if (info->hasDataToCommit && info->fset) {
-      tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
-    }
-  }
-  taosThreadMutexUnlock(&tsdb->mutex);
-
 _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);

From 444014f6ac0e88b9d4c04a5c5567e7cf8b819f20 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 29 May 2024 14:10:08 +0800
Subject: [PATCH 061/132] add ci

---
 tests/system-test/7-tmq/tmq_taosx.py | 51 ++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index 94e9babf3c..39b9c6d34a 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -278,6 +278,19 @@ class TDTestCase:
         self.checkDropData(False)
 
         return
+    
+    def checkSnapshot1VgroupBtmeta(self):
+        buildPath = tdCom.getBuildPath()
+        cfgPath = tdCom.getClientCfgPath()
+        cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -bt'%(buildPath, cfgPath)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+
+        self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot")
+        self.checkData()
+        self.checkDropData(False)
+
+        return
 
     def checkSnapshot1VgroupTable(self):
         buildPath = tdCom.getBuildPath()
@@ -290,6 +303,18 @@ class TDTestCase:
         self.checkDataTable()
 
         return
+    
+    def checkSnapshot1VgroupTableBtmeta(self):
+        buildPath = tdCom.getBuildPath()
+        cfgPath = tdCom.getClientCfgPath()
+        cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -t -bt'%(buildPath, cfgPath)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+
+        self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot")
+        self.checkDataTable()
+
+        return
 
     def checkSnapshotMultiVgroups(self):
         buildPath = tdCom.getBuildPath()
@@ -301,6 +326,17 @@ class TDTestCase:
         self.checkDropData(False)
 
         return
+    
+    def checkSnapshotMultiVgroupsBtmeta(self):
+        buildPath = tdCom.getBuildPath()
+        cmdStr = '%s/build/bin/tmq_taosx_ci -sv 2 -dv 4 -s -bt'%(buildPath)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+
+        self.checkData()
+        self.checkDropData(False)
+
+        return
 
     def checkSnapshotMultiVgroupsWithDropTable(self):
         buildPath = tdCom.getBuildPath()
@@ -311,6 +347,16 @@ class TDTestCase:
         self.checkDropData(True)
 
         return
+    
+    def checkSnapshotMultiVgroupsWithDropTableBtmeta(self):
+        buildPath = tdCom.getBuildPath()
+        cmdStr = '%s/build/bin/tmq_taosx_ci -sv 2 -dv 4 -s -d -bt'%(buildPath)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+
+        self.checkDropData(True)
+
+        return
 
     def consumeTest(self):
         tdSql.execute(f'create database if not exists d1 vgroups 1')
@@ -472,6 +518,11 @@ class TDTestCase:
 
         self.checkSnapshotMultiVgroupsWithDropTable()
 
+        self.checkSnapshot1VgroupBtmeta()
+        self.checkSnapshot1VgroupTableBtmeta()
+        self.checkSnapshotMultiVgroupsBtmeta()
+        self.checkSnapshotMultiVgroupsWithDropTableBtmeta()
+
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

From e75ab5b2985d3addb1fe522767443538b75a0c2a Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 29 May 2024 14:10:34 +0800
Subject: [PATCH 062/132] fix more

---
 source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 +-
 source/dnode/vnode/src/tsdb/tsdbFS2.c     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index 855bc12ff3..f6082e60e3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -624,7 +624,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
 
       int64_t minKey, maxKey;
       tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
-      tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
+      iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
       for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
         STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
         for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c
index 3087dc44d9..2345cc599b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c
@@ -1181,7 +1181,7 @@ int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) {
   int16_t sttTrigger = tsdb->pVnode->config.sttTrigger;
 
   tsdbFSGetFSet(tsdb->pFS, fid, fset);
-  if (sttTrigger == 1 && fset) {
+  if (sttTrigger == 1 && (*fset)) {
     for (;;) {
       if ((*fset)->taskRunning) {
         (*fset)->numWaitTask++;

From 27ac6ca16ed78ad5c0cb232e3e156098ebcc8717 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 14:12:14 +0800
Subject: [PATCH 063/132] fix(stream): desc the ref for checkpoint-trigger
 timer.

---
 include/libs/stream/tstream.h             |  1 +
 source/libs/stream/src/streamCheckpoint.c | 11 ++++++++---
 source/libs/stream/src/streamDispatch.c   |  9 +++++----
 source/libs/stream/src/streamQueue.c      | 12 ++++++++++--
 4 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 0d01913235..7e1c80c842 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -681,6 +681,7 @@ void    streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId);
 int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pRpcInfo);
 
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
+int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue);
 
 // common
 void    streamTaskPause(SStreamTask* pTask);
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index a177c56f77..3865d030f5 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -198,8 +198,9 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
     stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
     SActiveCheckpointInfo* pActive = pTask->chkInfo.pActiveInfo;
+    streamMetaAcquireOneTask(pTask);
+
     if (pActive->pCheckTmr == NULL) {
-      streamMetaAcquireOneTask(pTask);
       pActive->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
     } else {
       taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActive->pCheckTmr);
@@ -605,6 +606,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
   }
 
   if (++pActiveInfo->checkCounter < 100) {
+    taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pCheckTmr);
     return;
   }
 
@@ -614,7 +616,10 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
   taosThreadMutexLock(&pTask->lock);
   SStreamTaskState* pState = streamTaskGetStatus(pTask);
   if (pState->state != TASK_STATUS__CK) {
-    stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger", pTask->id.idStr, vgId);
+    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", pTask->id.idStr,
+            vgId, ref);
+
     taosThreadMutexUnlock(&pTask->lock);
     streamMetaReleaseTask(pTask->pMeta, pTask);
     return;
@@ -769,7 +774,7 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
   taosThreadMutexLock(&pInfo->lock);
 
   // outputQ should be empty here
-  ASSERT(streamQueueGetNumOfItems(pTask->outputq.queue) == 1);
+  ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0);
 
   pInfo->dispatchTrigger = true;
   if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index ecea494be2..0a8a65544c 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -566,9 +566,10 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
   const char* id = pTask->id.idStr;
   int32_t     numOfElems = streamQueueGetNumOfItems(pTask->outputq.queue);
   if (numOfElems > 0) {
-    double size = SIZE_IN_MiB(taosQueueMemorySize(pTask->outputq.queue->pQueue));
-    stDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d, size:%.2fMiB", id,
-            numOfElems, size);
+    double  size = SIZE_IN_MiB(taosQueueMemorySize(pTask->outputq.queue->pQueue));
+    int32_t numOfUnAccessed = streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue);
+    stDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d/%d, size:%.2fMiB", id,
+            numOfUnAccessed, numOfElems, size);
   }
 
   // to make sure only one dispatch is running
@@ -889,7 +890,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
 
   initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead));
 
-  stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64
+  stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64
           ":0x%x (vgId:%d) idx:%d, vgId:%d",
           pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index,
           req.upstreamNodeId);
diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c
index 247baea16f..461d53d5a9 100644
--- a/source/libs/stream/src/streamQueue.c
+++ b/source/libs/stream/src/streamQueue.c
@@ -117,6 +117,13 @@ int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue) {
   return numOfItems1 + numOfItems2;
 }
 
+int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue) {
+  int32_t numOfItems1 = taosQueueItemSize(pQueue->pQueue);
+  int32_t numOfItems2 = taosQallUnAccessedItemSize(pQueue->qall);
+
+  return numOfItems1 + numOfItems2;
+}
+
 int32_t streamQueueGetItemSize(const SStreamQueue* pQueue) {
   return taosQueueMemorySize(pQueue->pQueue) + taosQallUnAccessedMemSize(pQueue->qall);
 }
@@ -322,9 +329,10 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
     ASSERT(0);
   }
 
-  if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->info.triggerParam != 0) {
+  if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER &&
+      (pTask->info.triggerParam != 0)) {
     atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE);
-    stDebug("s-task:%s new data arrived, active the trigger, triggerStatus:%d", pTask->id.idStr, pTask->schedInfo.status);
+    stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, pTask->schedInfo.status);
   }
 
   return 0;

From 9e842929e81f25e192b2e2dd8eea10549e9b4a27 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Wed, 29 May 2024 14:32:21 +0800
Subject: [PATCH 064/132] TS-4847: add test case

---
 tests/system-test/2-query/interp.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py
index 81c5b66185..1cb95b59c5 100644
--- a/tests/system-test/2-query/interp.py
+++ b/tests/system-test/2-query/interp.py
@@ -15,6 +15,30 @@ class TDTestCase:
         #tdSql.init(conn.cursor())
         tdSql.init(conn.cursor(), logSql)  # output sql.txt file
 
+    def interp_on_empty_table(self):
+        dbname = "db"
+        tbname = "t"        
+
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+
+        tdSql.execute(f'''create table if not exists {dbname}.{tbname} (ts timestamp, k int)''')
+
+        tdLog.printNoPrefix("==========step2:interp query on empty table")
+
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(prev)")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(next)")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(linear)")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(value, 2)")        
+        tdSql.checkRows(0)
+    
     def run(self):
         dbname = "db"
         tbname = "tb"
@@ -5658,6 +5682,7 @@ class TDTestCase:
         tdSql.checkData(0,  0, '2023-08-06 23:59:00')
         tdSql.checkData(0,  1, None)
 
+        self.interp_on_empty_table()
 
 
     def stop(self):

From 003f9138bc145836707723ad2709076e3a5e9065 Mon Sep 17 00:00:00 2001
From: Chris Zhai <zk662144@163.com>
Date: Wed, 29 May 2024 14:59:22 +0800
Subject: [PATCH 065/132] add test cases for TS-4721

---
 tests/army/enterprise/alter/alterConfig.py | 55 ++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 tests/army/enterprise/alter/alterConfig.py

diff --git a/tests/army/enterprise/alter/alterConfig.py b/tests/army/enterprise/alter/alterConfig.py
new file mode 100644
index 0000000000..7413e5e5a6
--- /dev/null
+++ b/tests/army/enterprise/alter/alterConfig.py
@@ -0,0 +1,55 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+
+import taos
+import frame
+import frame.etool
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+
+
+class TDTestCase(TBase):
+    def alterSupportVnodes(self):
+        tdLog.info(f"test function of altering supportVnodes")
+
+        tdSql.execute("alter dnode 1 'supportVnodes' '128'")
+        time.sleep(1)
+        tdSql.query('show dnodes')
+        tdSql.checkData(0, 3, "128")
+        
+        tdSql.execute("alter dnode 1 'supportVnodes' '64'")
+        time.sleep(1)
+        tdSql.query('show dnodes')
+        tdSql.checkData(0, 3, "64")
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # TS-4721
+        self.alterSupportVnodes()
+
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

From c9c1e3ec9d1dba9de4822becc5db3c476543bca6 Mon Sep 17 00:00:00 2001
From: Chris Zhai <zk662144@163.com>
Date: Wed, 29 May 2024 15:01:18 +0800
Subject: [PATCH 066/132] update cases.task to add test cases

---
 tests/parallel_test/cases.task | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index e989cb20c7..81b720829c 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -25,6 +25,7 @@
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
 ,,n,army,python3 ./test.py -f community/query/show.py -N 3
+,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3
 

From 46ca55ac749402879aac775904757866d50a7c88 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 15:20:35 +0800
Subject: [PATCH 067/132] fix(stream): avoid deadlock.

---
 source/libs/stream/src/streamCheckpoint.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 3865d030f5..76bf139c3f 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -309,12 +309,15 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin
 
     { // destroy the related fill-history tasks
       // drop task should not in the meta-lock, and drop the related fill-history task now
+      streamMetaWUnLock(pMeta);
       if (pReq->dropRelHTask) {
         streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
         int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
         stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d",
                 id, vgId, pReq->taskId, numOfTasks);
       }
+
+      streamMetaWLock(pMeta);
     }
 
     return TSDB_CODE_SUCCESS;

From 1a6a0d31070d1143caccbe78a8c6eb279f9bb975 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 29 May 2024 16:04:37 +0800
Subject: [PATCH 068/132] fix:[TD-30323]tmq close error if consumer is cleared

---
 source/client/src/clientTmq.c             | 5 ++++-
 source/dnode/mnode/impl/src/mndConsumer.c | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 3141e21f26..0446af3226 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1260,7 +1260,10 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   int32_t retryCnt = 0;
   while ((code = syncAskEp(tmq)) != 0) {
     if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
-      tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId);
+      tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", tmq->consumerId, strerror(code));
+      if(code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
+        code = 0;
+      }
       goto FAIL;
     }
 
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index c7ae941389..9278fbce11 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -662,6 +662,10 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
   SMqConsumerObj *pConsumerNew = NULL;
   STrans         *pTrans = NULL;
 
+  if(mndAcquireConsumer(pMnode, subscribe->consumerId) == NULL && taosArrayGetSize(subscribe->topicNames) == 0){
+    goto _over;
+  }
+
   code = checkAndSortTopic(pMnode, subscribe.topicNames);
   if(code != TSDB_CODE_SUCCESS){
     goto _over;

From 129d04a4495f1a01d390b2c8ee1e7192484a0ef1 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 29 May 2024 16:29:04 +0800
Subject: [PATCH 069/132] adj tmq param

---
 source/dnode/vnode/src/tq/tqUtil.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 496201e79e..33e3414a7d 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -283,7 +283,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
 
         tqDebug("fetch meta msg, ver:%" PRId64 ", vgId:%d, type:%s, enable batch meta:%d", pHead->version, vgId,
                 TMSG_INFO(pHead->msgType), pRequest->enableBatchMeta);
-        if (!pRequest->enableBatchMeta) {
+        if (!pRequest->enableBatchMeta && !pRequest->useSnapshot) {
           SMqMetaRsp metaRsp = {0};
           tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
           metaRsp.resMsgType = pHead->msgType;

From a98394949ab3de03d6abd428e380aaa031e54866 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 29 May 2024 17:20:56 +0800
Subject: [PATCH 070/132] fix: add ut thread join

---
 source/libs/scheduler/test/schedulerTests.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp
index c52a8599a0..6d2215264e 100644
--- a/source/libs/scheduler/test/schedulerTests.cpp
+++ b/source/libs/scheduler/test/schedulerTests.cpp
@@ -798,6 +798,7 @@ TEST(queryTest, normalCase) {
 
   schedulerFreeJob(&job, 0);
 
+  taosThreadJoin(thread1, NULL);
 }
 
 TEST(queryTest, readyFirstCase) {
@@ -907,6 +908,8 @@ TEST(queryTest, readyFirstCase) {
   schedulerDestroy();
 
   schedulerFreeJob(&job, 0);
+
+  taosThreadJoin(thread1, NULL);
 }
 
 TEST(queryTest, flowCtrlCase) {
@@ -1001,6 +1004,8 @@ TEST(queryTest, flowCtrlCase) {
   schedulerDestroy();
 
   schedulerFreeJob(&job, 0);
+
+  taosThreadJoin(thread1, NULL);  
 }
 
 TEST(insertTest, normalCase) {
@@ -1061,6 +1066,8 @@ TEST(insertTest, normalCase) {
   schedulerFreeJob(&insertJobRefId, 0);
 
   schedulerDestroy();
+
+  taosThreadJoin(thread1, NULL);  
 }
 
 TEST(multiThread, forceFree) {

From 4f49cb7940bd9516e084f933e5579d0e510b7582 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Wed, 29 May 2024 17:29:31 +0800
Subject: [PATCH 071/132] coverage: add test case for db encrypt

---
 tests/army/enterprise/db-encrypt/basic.py | 63 +++++++++++++++++++++++
 tests/parallel_test/cases.task            |  1 +
 2 files changed, 64 insertions(+)
 create mode 100644 tests/army/enterprise/db-encrypt/basic.py

diff --git a/tests/army/enterprise/db-encrypt/basic.py b/tests/army/enterprise/db-encrypt/basic.py
new file mode 100644
index 0000000000..8d30bbcfe2
--- /dev/null
+++ b/tests/army/enterprise/db-encrypt/basic.py
@@ -0,0 +1,63 @@
+import taos
+import sys
+import os
+import subprocess
+import glob
+import shutil
+import time
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.srvCtl import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+# from frame.server.dnodes import *
+# from frame.server.cluster import *
+
+
+class TDTestCase(TBase):
+    
+    def init(self, conn, logSql, replicaVar=1):
+        super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1")
+        self.valgrind = 0
+        self.db = "test"
+        self.stb = "meters"
+        self.childtable_count = 10
+        tdSql.init(conn.cursor(), logSql)  
+    
+    def create_encrypt_db(self):        
+        
+        tdSql.execute("create encrypt_key '1234567890'")
+        autoGen = AutoGen()
+        autoGen.create_db(self.db, 2, 1, "ENCRYPT_ALGORITHM 'sm4'")
+        tdSql.execute(f"use {self.db}")
+        autoGen.create_stable(self.stb, 2, 3, 8, 8)
+        autoGen.create_child(self.stb, "d", self.childtable_count)
+        autoGen.insert_data(1000)
+        
+        tdSql.query(f"select * from {self.db}.{self.stb}")
+        tdSql.checkRows(1000 * self.childtable_count)
+        
+        self.timestamp_step = 1000
+        self.insert_rows = 1000
+        
+        self.checkInsertCorrect()           
+    
+    def create_encrypt_db_error(self):
+        tdSql.error("create encrypt_key '123'")
+        tdSql.error("create encrypt_key '12345678abcdefghi'")
+        tdSql.error("create database test ENCRYPT_ALGORITHM 'sm4'")
+
+    def run(self):
+        self.create_encrypt_db_error()
+        self.create_encrypt_db()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index e989cb20c7..e29f5ca6bf 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -11,6 +11,7 @@
 # army-test
 #
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f enterprise/db-encrypt/basic.py
 ,,n,army,python3 ./test.py -f enterprise/s3/s3Basic.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
 ,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py

From becc4cb368cd29831fe36816ecc5161c4111d03e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 19:29:26 +0800
Subject: [PATCH 072/132] enh(stream): limit the concurrent started checkpoint
 tasks.

---
 include/common/tglobal.h                     |   1 +
 source/common/src/tglobal.c                  | 230 ++++++-------------
 source/dnode/mnode/impl/inc/mndStream.h      |   1 +
 source/dnode/mnode/impl/src/mndMain.c        |  52 +----
 source/dnode/mnode/impl/src/mndStream.c      | 150 ++++++------
 source/dnode/mnode/impl/src/mndStreamTrans.c |  41 ++--
 source/libs/stream/src/streamCheckpoint.c    |   2 -
 7 files changed, 195 insertions(+), 282 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 95b7591263..90ee6f7cc0 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -235,6 +235,7 @@ extern int32_t tsMqRebalanceInterval;
 extern int32_t tsStreamCheckpointInterval;
 extern float   tsSinkDataRate;
 extern int32_t tsStreamNodeCheckInterval;
+extern int32_t tsMaxConcurrentCheckpoint;
 extern int32_t tsTtlUnit;
 extern int32_t tsTtlPushIntervalSec;
 extern int32_t tsTtlBatchDropNum;
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 691eccd174..2fefeb4cf2 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -274,6 +274,7 @@ int32_t tsMqRebalanceInterval = 2;
 int32_t tsStreamCheckpointInterval = 60;
 float   tsSinkDataRate = 2.0;
 int32_t tsStreamNodeCheckInterval = 16;
+int32_t tsMaxConcurrentCheckpoint = 1;
 int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushIntervalSec = 10;
 int32_t tsTrimVDbIntervalSec = 60 * 60;    // interval of trimming db in all vgroups
@@ -621,114 +622,80 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
 }
 
 static int32_t taosAddServerCfg(SConfig *pCfg) {
-  if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  tsNumOfCommitThreads = tsNumOfCores / 2;
+  tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
 
   tsNumOfSupportVnodes = tsNumOfCores * 2 + 5;
   tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
-  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-
-  if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  // if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-
-  if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
-    return -1;
-
-  if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
-
-  tsNumOfCommitThreads = tsNumOfCores / 2;
-  tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
-  if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
 
   tsNumOfMnodeReadThreads = tsNumOfCores / 8;
   tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4);
-  if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
 
   tsNumOfVnodeQueryThreads = tsNumOfCores * 2;
   tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 16);
-  if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) !=
-      0)
-    return -1;
-
-  if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
 
   tsNumOfVnodeFetchThreads = tsNumOfCores / 4;
   tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
-  if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) !=
-      0)
-    return -1;
 
   tsNumOfVnodeRsmaThreads = tsNumOfCores / 4;
   tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4);
-  if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
 
   tsNumOfQnodeQueryThreads = tsNumOfCores * 2;
   tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 16);
-  if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) !=
-      0)
-    return -1;
+
+  tsNumOfSnodeStreamThreads = tsNumOfCores / 4;
+  tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4);
+
+  tsNumOfSnodeWriteThreads = tsNumOfCores / 4;
+  tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4);
+
+  tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1;
+  tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL);
+
+  // clang-format off
+  if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+
+  if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+
+  if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
+  if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
+
+  if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+
+  if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+
+  if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   //  tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
   //  tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
   //  if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
 
-  tsNumOfSnodeStreamThreads = tsNumOfCores / 4;
-  tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4);
-  if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
-  tsNumOfSnodeWriteThreads = tsNumOfCores / 4;
-  tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4);
-  if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) !=
-      0)
-    return -1;
+  if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
 
-  tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1;
-  tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL);
-  if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX,
-                  CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
-  if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) !=
-      0)
-    return -1;
-  if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2),
-                  CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
-  if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
-
-  if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-  if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@@ -736,9 +703,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER,
-                  CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
@@ -752,68 +717,43 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   if (cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1;
   if (cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
-  if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0)
-    return -1;
+  if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
 
-  if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
   if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
   if (cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
-      0)
-    return -1;
-  if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER,
-                  CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
-      0)
-    return -1;
+  if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+
   if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-  if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
-      0)
-    return -1;
-  if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
-      0)
-    return -1;
-  if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER,
-                  CFG_DYN_ENT_SERVER) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
-      0)
-    return -1;
+  if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
-  if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX,
-                  CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
-  if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
-  if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
-                  CFG_DYN_ENT_SERVER) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
-  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER,
-                  CFG_DYN_ENT_SERVER) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
   if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@@ -823,41 +763,23 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER,
-                  CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  /*
-  if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-  if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) {
-    uError("failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", tsS3BlockSize);
-    return -1;
-  }
-  if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
-      0)
-    return -1;
-  */
-  if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
-                  CFG_DYN_ENT_SERVER) != 0)
-    return -1;
-  if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
-                  CFG_DYN_ENT_SERVER) != 0)
-    return -1;
+
+  if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
   // min free disk space used to check if the disk is full [50MB, 1GB]
-  if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024,
-                  CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
-    return -1;
+  if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
+  // clang-format on
+
   // GRANT_CFG_ADD;
   return 0;
 }
diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index 5a4caf3348..bb10a9b9ad 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -97,6 +97,7 @@ void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
 int32_t     mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
 int32_t     mndPersistStream(STrans *pTrans, SStreamObj *pStream);
 int32_t     mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId);
+int32_t     mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt);
 int32_t     mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId);
 bool        mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock);
 int32_t     mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId);
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index a78edcb05e..a228166124 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -90,21 +90,6 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
   return pReq;
 }
 
-static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) {
-  SMStreamTickReq timerReq = {
-      .tick = sec,
-  };
-
-  int32_t contLen = tSerializeSMStreamTickMsg(NULL, 0, &timerReq);
-  if (contLen <= 0) return NULL;
-  void *pReq = rpcMallocCont(contLen);
-  if (pReq == NULL) return NULL;
-
-  tSerializeSMStreamTickMsg(pReq, contLen, &timerReq);
-  *pContLen = contLen;
-  return pReq;
-}
-
 static void mndPullupTrans(SMnode *pMnode) {
   mTrace("pullup trans msg");
   int32_t contLen = 0;
@@ -174,21 +159,12 @@ static void mndCalMqRebalance(SMnode *pMnode) {
   }
 }
 
-static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) {
-  int32_t contLen = 0;
-  void   *pReq = mndBuildCheckpointTickMsg(&contLen, sec);
-  if (pReq != NULL) {
-    SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER, .pCont = pReq, .contLen = contLen};
-    tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
-  }
-}
-
-static void mndStreamCheckpointRemain(SMnode *pMnode) {
-  int32_t contLen = 0;
-  void   *pReq = mndBuildCheckpointTickMsg(&contLen, 0);
-  if (pReq != NULL) {
-    SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen};
-    tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
+static void mndStreamCheckpointTimer(SMnode *pMnode) {
+  SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
+  if (pMsg != NULL) {
+    int32_t size = sizeof(SMStreamDoCheckpointMsg);
+    SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
   }
 }
 
@@ -367,12 +343,8 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
     mndCalMqRebalance(pMnode);
   }
 
-  if (sec % tsStreamCheckpointInterval == 0) {
-    mndStreamCheckpointTick(pMnode, sec);
-  }
-
-  if (sec % 5 == 0) {
-    mndStreamCheckpointRemain(pMnode);
+  if (sec % 30 == 0) {  // send the checkpoint info every 10 sec
+    mndStreamCheckpointTimer(pMnode);
   }
 
   if (sec % tsStreamNodeCheckInterval == 0) {
@@ -399,6 +371,7 @@ void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) {
     mndSyncCheckTimeout(pMnode);
   }
 }
+
 static void *mndThreadFp(void *param) {
   SMnode *pMnode = param;
   int64_t lastTime = 0;
@@ -832,10 +805,9 @@ _OVER:
       pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
       pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER ||
       pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER ||
-      pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE ||
-      pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT ||
-      pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER ||
-      pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
+      pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT ||
+      pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER || pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER ||
+      pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
     mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
            pMnode->stopped, state.restored, syncStr(state.state));
     return -1;
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 989d9970cd..ef29547e20 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -45,9 +45,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq);
 static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq);
 
-static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
 static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
-static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq);
 static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextStream(SMnode *pMnode, void *pIter);
 static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
@@ -114,10 +112,8 @@ int32_t mndInitStream(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_STREAM_DROP_RSP, mndTransProcessRsp);
 
   mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
-  mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
   mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
   mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint);
-  mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, mndProcessStreamCheckpointInCandid);
   mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
   mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq);
@@ -886,26 +882,10 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
     }
   }
 
-  mDebug("generated checkpoint %" PRId64 "", maxChkptId + 1);
+  mDebug("generate new checkpointId:%" PRId64, maxChkptId + 1);
   return maxChkptId + 1;
 }
 
-static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
-  SMnode *pMnode = pReq->info.node;
-  SSdb   *pSdb = pMnode->pSdb;
-  if (sdbGetSize(pSdb, SDB_STREAM) <= 0) {
-    return 0;
-  }
-
-  SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
-  pMsg->checkpointId = mndStreamGenChkptId(pMnode, true);
-
-  int32_t size = sizeof(SMStreamDoCheckpointMsg);
-  SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
-  tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
-  return 0;
-}
-
 static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId,
                                                  int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger) {
   SStreamCheckpointSourceReq req = {0};
@@ -1136,73 +1116,101 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) {
   return ready ? 0 : -1;
 }
 
+typedef struct {
+  int64_t streamId;
+  int64_t duration;
+} SCheckpointInterval;
+
+static int32_t streamWaitComparFn(const void* p1, const void* p2) {
+  const SCheckpointInterval* pInt1 = p1;
+  const SCheckpointInterval* pInt2 = p2;
+  if (pInt1->duration == pInt2->duration) {
+    return 0;
+  }
+
+  return pInt1->duration > pInt2->duration? -1:1;
+}
+
 static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
   SMnode     *pMnode = pReq->info.node;
   SSdb       *pSdb = pMnode->pSdb;
   void       *pIter = NULL;
   SStreamObj *pStream = NULL;
   int32_t     code = 0;
+  int32_t     numOfCheckpointTrans = 0;
 
   if ((code = mndCheckNodeStatus(pMnode)) != 0) {
     return code;
   }
 
-  // make sure the time interval between two consecutive checkpoint trans is long enough
-  SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont;
+  SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval));
+  int64_t now = taosGetTimestampMs();
   while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) {
-    code = mndProcessStreamCheckpointTrans(pMnode, pStream, pMsg->checkpointId, 1, true);
-    sdbRelease(pSdb, pStream);
-    if (code == -1) {
-      break;
-    }
-  }
-
-  return code;
-}
-
-static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) {
-  SMnode *pMnode = pReq->info.node;
-  void   *pIter = NULL;
-  int32_t code = 0;
-
-  taosThreadMutexLock(&execInfo.lock);
-  int32_t num = taosHashGetSize(execInfo.transMgmt.pWaitingList);
-  taosThreadMutexUnlock(&execInfo.lock);
-  if (num == 0) {
-    return code;
-  }
-
-  if ((code = mndCheckNodeStatus(pMnode)) != 0) {
-    return code;
-  }
-
-  SArray *pList = taosArrayInit(4, sizeof(int64_t));
-  while ((pIter = taosHashIterate(execInfo.transMgmt.pWaitingList, pIter)) != NULL) {
-    SCheckpointCandEntry *pEntry = pIter;
-
-    SStreamObj *ps = mndAcquireStream(pMnode, pEntry->pName);
-    if (ps == NULL) {
+    int64_t duration = now - pStream->checkpointFreq;
+    if (duration < tsStreamCheckpointInterval * 1000) {
+      sdbRelease(pSdb, pStream);
       continue;
     }
 
-    mDebug("start to launch checkpoint for stream:%s %" PRIx64 " in candidate list", pEntry->pName, pEntry->streamId);
+    SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration};
+    taosArrayPush(pList, &in);
 
-    code = mndProcessStreamCheckpointTrans(pMnode, ps, pEntry->checkpointId, 1, true);
-    mndReleaseStream(pMnode, ps);
+    int32_t currentSize = taosArrayGetSize(pList);
+    mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %" PRId64 "s(%" PRId64
+           "s) beyond threshold:%d",
+           pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize);
 
-    if (code == TSDB_CODE_SUCCESS) {
-      taosArrayPush(pList, &pEntry->streamId);
+    sdbRelease(pSdb, pStream);
+  }
+
+  int32_t size = taosArrayGetSize(pList);
+  if (size == 0) {
+    taosArrayDestroy(pList);
+    return code;
+  }
+
+  taosArraySort(pList, streamWaitComparFn);
+  mndStreamClearFinishedTrans(pMnode, &numOfCheckpointTrans);
+  int32_t numOfQual = taosArrayGetSize(pList);
+
+  if (numOfCheckpointTrans > tsMaxConcurrentCheckpoint) {
+    mDebug(
+        "%d stream(s) checkpoint interval longer than %ds, ongoing checkpoint trans:%d reach maximum allowed:%d, new "
+        "checkpoint trans are not allowed, wait for 30s",
+        numOfQual, tsStreamCheckpointInterval, numOfCheckpointTrans, tsMaxConcurrentCheckpoint);
+    taosArrayDestroy(pList);
+    return code;
+  }
+
+  int32_t capacity = tsMaxConcurrentCheckpoint - numOfCheckpointTrans;
+  mDebug(
+      "%d stream(s) checkpoint interval longer than %ds, %d ongoing checkpoint trans, %d new checkpoint trans allowed, "
+      "concurrent trans threshold:%d",
+      numOfQual, tsStreamCheckpointInterval, numOfCheckpointTrans, capacity, tsMaxConcurrentCheckpoint);
+
+  int32_t started = 0;
+  int64_t checkpointId = mndStreamGenChkptId(pMnode, true);
+
+  for (int32_t i = 0; i < numOfQual; ++i) {
+    SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i);
+
+    SStreamObj *p = mndGetStreamObj(pMnode, pCheckpointInfo->streamId);
+    if (p != NULL) {
+      code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true);
+      sdbRelease(pSdb, p);
+
+      if (code != -1) {
+        started += 1;
+
+        if (started >= capacity) {
+          mDebug("already start %d new checkpoint trans, current active checkpoint trans:%d", started,
+                 (started + numOfCheckpointTrans));
+          break;
+        }
+      }
     }
   }
 
-  for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
-    int64_t *pId = taosArrayGet(pList, i);
-
-    taosHashRemove(execInfo.transMgmt.pWaitingList, pId, sizeof(*pId));
-  }
-
-  int32_t remain = taosHashGetSize(execInfo.transMgmt.pWaitingList);
-  mDebug("%d in candidate list generated checkpoint, remaining:%d", (int32_t)taosArrayGetSize(pList), remain);
   taosArrayDestroy(pList);
   return code;
 }
@@ -2349,10 +2357,10 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
     return 0;
   }
 
-  SMStreamNodeCheckMsg *pMsg = rpcMallocCont(sizeof(SMStreamNodeCheckMsg));
+  int32_t size = sizeof(SMStreamNodeCheckMsg);
+  SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size);
 
-  SRpcMsg rpcMsg = {
-      .msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = sizeof(SMStreamNodeCheckMsg)};
+  SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size};
   tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
   return 0;
 }
diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c
index 74ad09c752..ac4cb08308 100644
--- a/source/dnode/mnode/impl/src/mndStreamTrans.c
+++ b/source/dnode/mnode/impl/src/mndStreamTrans.c
@@ -21,8 +21,6 @@ typedef struct SKeyInfo {
   int32_t keyLen;
 } SKeyInfo;
 
-static int32_t clearFinishedTrans(SMnode* pMnode);
-
 int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamId) {
   SStreamTransInfo info = {
       .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId};
@@ -30,41 +28,54 @@ int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t s
   return 0;
 }
 
-int32_t clearFinishedTrans(SMnode* pMnode) {
+int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) {
   size_t  keyLen = 0;
   void*   pIter = NULL;
   SArray* pList = taosArrayInit(4, sizeof(SKeyInfo));
+  int32_t num = 0;
 
   while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) {
-    SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter;
+    SStreamTransInfo *pEntry = (SStreamTransInfo *)pIter;
 
     // let's clear the finished trans
-    STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId);
+    STrans *pTrans = mndAcquireTrans(pMnode, pEntry->transId);
     if (pTrans == NULL) {
-      void* pKey = taosHashGetKey(pEntry, &keyLen);
+      void *pKey = taosHashGetKey(pEntry, &keyLen);
       // key is the name of src/dst db name
       SKeyInfo info = {.pKey = pKey, .keyLen = keyLen};
-      mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name,
-             pEntry->startTime);
+      mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, pEntry->startTime);
       taosArrayPush(pList, &info);
     } else {
+      if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) {
+        num++;
+      }
       mndReleaseTrans(pMnode, pTrans);
     }
   }
 
-  size_t num = taosArrayGetSize(pList);
-  for (int32_t i = 0; i < num; ++i) {
+  int32_t size = taosArrayGetSize(pList);
+  for (int32_t i = 0; i < size; ++i) {
     SKeyInfo* pKey = taosArrayGet(pList, i);
     taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen);
   }
 
-  mDebug("clear %d finished stream-trans, remained:%d", (int32_t)num, taosHashGetSize(execInfo.transMgmt.pDBTrans));
+  mDebug("clear %d finished stream-trans, remained:%d, active checkpoint trans:%d", size,
+         taosHashGetSize(execInfo.transMgmt.pDBTrans), num);
 
   terrno = TSDB_CODE_SUCCESS;
   taosArrayDestroy(pList);
+
+  if (pNumOfActiveChkpt != NULL) {
+    *pNumOfActiveChkpt = num;
+  }
+
   return 0;
 }
 
+// * Transactions of different streams are not related. Here only check the conflict of transaction for a given stream.
+// For a given stream:
+// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans.
+// 2. create/drop/reset/update trans are conflict with any other trans.
 bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* pTransName, bool lock) {
   if (lock) {
     taosThreadMutexLock(&execInfo.lock);
@@ -78,7 +89,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
     return false;
   }
 
-  clearFinishedTrans(pMnode);
+  mndStreamClearFinishedTrans(pMnode, NULL);
 
   SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
   if (pEntry != NULL) {
@@ -95,7 +106,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
         terrno = TSDB_CODE_MND_TRANS_CONFLICT;
         return true;
       } else {
-        mDebug("not conflict with checkpoint trans, name:%s, continue create trans", pTransName);
+        mDebug("not conflict with checkpoint trans, name:%s, continue creating trans", pTransName);
       }
     } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) ||
                (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0) ||
@@ -106,7 +117,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
       return true;
     }
   } else {
-    mDebug("stream:0x%"PRIx64" no conflict trans existed, continue create trans", streamId);
+    mDebug("stream:0x%" PRIx64 " no conflict trans existed, continue create trans", streamId);
   }
 
   if (lock) {
@@ -124,7 +135,7 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) {
     return 0;
   }
 
-  clearFinishedTrans(pMnode);
+  mndStreamClearFinishedTrans(pMnode, NULL);
   SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid));
   if (pEntry != NULL) {
     SStreamTransInfo tInfo = *pEntry;
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 76bf139c3f..6b02ae485f 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -377,8 +377,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin
   // drop task should not in the meta-lock, and drop the related fill-history task now
   if (pReq->dropRelHTask) {
     streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
-
-    // commit the update
     int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
     stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, pReq->taskId, numOfTasks);
   }

From 0946f25e5f1ea19fe559022cd41cf211091b02d1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 29 May 2024 21:59:26 +0800
Subject: [PATCH 073/132] fix(stream): fix syntax error.

---
 source/dnode/mnode/impl/src/mndStream.c | 2 +-
 source/libs/stream/src/streamDispatch.c | 2 +-
 source/libs/stream/src/streamExec.c     | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index ef29547e20..09e0e4e415 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1156,7 +1156,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
     taosArrayPush(pList, &in);
 
     int32_t currentSize = taosArrayGetSize(pList);
-    mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %" PRId64 "s(%" PRId64
+    mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64
            "s) beyond threshold:%d",
            pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize);
 
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 0a8a65544c..522ac9a910 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -1038,7 +1038,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
       taosArrayPush(pTask->msgInfo.pRetryList, &pRsp->downstreamNodeId);
       taosThreadMutexUnlock(&pTask->lock);
 
-      stWarn("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for %dms and retry dispatch", id,
+      stTrace("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for %dms and retry dispatch", id,
              pRsp->downstreamTaskId, pRsp->downstreamNodeId, DISPATCH_RETRY_INTERVAL_MS);
     } else if (pRsp->inputStatus == TASK_INPUT_STATUS__REFUSED) {
       // todo handle the agg task failure, add test case
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 2b92696b88..77f266bd6d 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -562,13 +562,13 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
     }
 
     if (streamQueueIsFull(pTask->outputq.queue)) {
-      stWarn("s-task:%s outputQ is full, idle for 500ms and retry", id);
+      stTrace("s-task:%s outputQ is full, idle for 500ms and retry", id);
       streamTaskSetIdleInfo(pTask, 1000);
       return 0;
     }
 
     if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
-      stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", id);
+      stTrace("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", id);
       streamTaskSetIdleInfo(pTask, 1000);
       return 0;
     }

From 88a2c373b35aa849c8c552df4a6add0aa08fd5df Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 30 May 2024 10:22:26 +0800
Subject: [PATCH 074/132] more fix

---
 source/dnode/vnode/src/tsdb/tsdbCommit2.c | 80 ++++++++---------------
 1 file changed, 29 insertions(+), 51 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index f6082e60e3..551d4c62b2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -18,7 +18,6 @@
 // extern dependencies
 typedef struct {
   int32_t    fid;
-  bool       hasDataToCommit;
   STFileSet *fset;
 } SFileSetCommitInfo;
 
@@ -512,36 +511,25 @@ _exit:
   return code;
 }
 
-static int32_t tsdbCommitInfoAdd(STsdb *tsdb, const SFileSetCommitInfo *info) {
+static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) {
   int32_t code = 0;
   int32_t lino = 0;
 
   SFileSetCommitInfo *tinfo;
 
-  vHashGet(tsdb->commitInfo->ht, info, (void **)&tinfo);
-  if (tinfo) {
-    if (info->hasDataToCommit && !tinfo->hasDataToCommit) {
-      tinfo->hasDataToCommit = true;
-    }
-  } else {
-    if ((tinfo = taosMemoryCalloc(1, sizeof(*tinfo))) == NULL) {
-      TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
-    }
-    tinfo->fid = info->fid;
-    tinfo->hasDataToCommit = info->hasDataToCommit;
-    if (info->fset) {
-      code = tsdbTFileSetInitCopy(tsdb, info->fset, &tinfo->fset);
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    code = vHashPut(tsdb->commitInfo->ht, tinfo);
-    TSDB_CHECK_CODE(code, lino, _exit);
-
-    if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) {
-      TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
-    }
-    taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare);
+  if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
   }
+  tinfo->fid = fid;
+  tinfo->fset = NULL;
+
+  code = vHashPut(tsdb->commitInfo->ht, tinfo);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+  }
+  taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare);
 
 _exit:
   if (code) {
@@ -585,12 +573,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
       fid = tsdbKeyFid(TSDBROW_TS(row), tsdb->keepCfg.days, tsdb->keepCfg.precision);
       tsdbFidKeyRange(fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
 
-      SFileSetCommitInfo info = {
-          .fid = fid,
-          .hasDataToCommit = true,
-          .fset = NULL,
-      };
-      code = tsdbCommitInfoAdd(tsdb, &info);
+      code = tsdbCommitInfoAdd(tsdb, fid);
       TSDB_CHECK_CODE(code, lino, _exit);
 
       from.key.ts = maxKey + 1;
@@ -599,18 +582,13 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
 
   taosThreadMutexLock(&tsdb->mutex);
 
-  // copy existing file set
-  for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
-    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
-    tsdbFSGetFSet(tsdb->pFS, info->fid, &fset);
-    if (fset) {
-      tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
-    }
-  }
-
   // scan tomb data
   if (tsdb->imem->nDel > 0) {
     TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
+      if (tsdbTFileSetIsEmpty(fset)) {
+        continue;
+      }
+
       SFileSetCommitInfo *info;
       SFileSetCommitInfo  tinfo = {
            .fid = fset->fid,
@@ -623,6 +601,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
       }
 
       int64_t minKey, maxKey;
+      bool    hasDataToCommit = false;
       tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
       iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
       for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
@@ -631,10 +610,8 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
           if (pDelData->sKey > maxKey || pDelData->eKey < minKey) {
             continue;
           } else {
-            tinfo.fid = fset->fid;
-            tinfo.hasDataToCommit = true;
-            tinfo.fset = fset;
-            if ((code = tsdbCommitInfoAdd(tsdb, &tinfo))) {
+            hasDataToCommit = true;
+            if ((code = tsdbCommitInfoAdd(tsdb, fset->fid))) {
               taosThreadMutexUnlock(&tsdb->mutex);
               TSDB_CHECK_CODE(code, lino, _exit);
             }
@@ -642,7 +619,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
           }
         }
 
-        if (tinfo.hasDataToCommit) {
+        if (hasDataToCommit) {
           break;
         }
       }
@@ -653,6 +630,9 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
   for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
     SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
     tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
+    if (fset) {
+      tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
+    }
   }
 
   taosThreadMutexUnlock(&tsdb->mutex);
@@ -756,10 +736,8 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
 
     for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
       committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
-      if (committer.ctx->info->hasDataToCommit) {
-        code = tsdbCommitFileSet(&committer);
-        TSDB_CHECK_CODE(code, lino, _exit);
-      }
+      code = tsdbCommitFileSet(&committer);
+      TSDB_CHECK_CODE(code, lino, _exit);
     }
 
     code = tsdbCloseCommitter(&committer, code);
@@ -792,7 +770,7 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
 
     for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
       SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
-      if (info->hasDataToCommit && info->fset) {
+      if (info->fset) {
         tsdbFinishTaskOnFileSet(tsdb, info->fid);
       }
     }
@@ -824,7 +802,7 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) {
   taosThreadMutexLock(&pTsdb->mutex);
   for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
     SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
-    if (info->hasDataToCommit && info->fset) {
+    if (info->fset) {
       tsdbFinishTaskOnFileSet(pTsdb, info->fid);
     }
   }

From 91394f2e6dabc2bf4e61b85ccc47c6ae805f9b93 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 30 May 2024 11:01:44 +0800
Subject: [PATCH 075/132] fix more

---
 source/dnode/vnode/src/tsdb/tsdbCommit2.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index 551d4c62b2..ffe5c2c1e0 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -573,8 +573,15 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
       fid = tsdbKeyFid(TSDBROW_TS(row), tsdb->keepCfg.days, tsdb->keepCfg.precision);
       tsdbFidKeyRange(fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
 
-      code = tsdbCommitInfoAdd(tsdb, fid);
-      TSDB_CHECK_CODE(code, lino, _exit);
+      SFileSetCommitInfo *info;
+      SFileSetCommitInfo  tinfo = {
+           .fid = fid,
+      };
+      vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
+      if (info == NULL) {
+        code = tsdbCommitInfoAdd(tsdb, fid);
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
 
       from.key.ts = maxKey + 1;
     }

From cff5e753ef28b2656b603f1723f9764a0e2292ba Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 30 May 2024 14:12:23 +0800
Subject: [PATCH 076/132] fix(stream): discard the result data block if the
 primary timestamp of results is expired.

---
 source/dnode/vnode/src/inc/tq.h           |  2 +-
 source/dnode/vnode/src/inc/vnodeInt.h     |  1 +
 source/dnode/vnode/src/sma/smaTimeRange.c |  4 +-
 source/dnode/vnode/src/tq/tqSink.c        | 50 ++++++++++++++++-------
 source/dnode/vnode/src/tsdb/tsdbOpen.c    |  8 ++++
 source/dnode/vnode/src/tsdb/tsdbRead2.c   | 10 ++---
 6 files changed, 50 insertions(+), 25 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index e2ecdca59f..338a0b40bd 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -160,7 +160,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp,
 int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset);
 void    tqUpdateNodeStage(STQ* pTq, bool isLeader);
 int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
-                                 SSubmitTbData* pTableData, const char* id);
+                                 SSubmitTbData* pTableData, int64_t earlyTs, const char* id);
 int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
 
 SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index b369bd6039..8222af4d60 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -242,6 +242,7 @@ int     tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmit
 int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
 int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
+int64_t tsdbGetEarliestTs(STsdb* pTsdb);
 
 // tq
 STQ*    tqOpen(const char* path, SVnode* pVnode);
diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c
index fb898c02f8..d0913081ac 100644
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@@ -203,7 +203,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
 
       int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
       if (index == NULL) {  // no data yet, append it
-        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
+        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, INT64_MIN, "");
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }
@@ -213,7 +213,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
         int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1;
         taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
       } else {
-        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
+        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, INT64_MIN, "");
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 5929c6b591..3fe124ea85 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -34,7 +34,7 @@ static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSData
 static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks);
 static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen);
 static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock,
-                             const char* id);
+                             int64_t earlyTs, const char* id);
 static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo,
                                         const char* dstTableName, int64_t* uid);
 static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId,
@@ -552,7 +552,8 @@ int32_t tsAscendingSortFn(const void* p1, const void* p2) {
   }
 }
 
-int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
+int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, int64_t earlyTs,
+                      const char* id) {
   int32_t numOfRows = pDataBlock->info.rows;
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -581,6 +582,14 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
         SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex);
         ts = *(int64_t*)colDataGetData(pColData, j);
         tqTrace("s-task:%s sink row %d, col %d ts %" PRId64, id, j, k, ts);
+
+        if (ts < earlyTs) {
+          tqError("s-task:%s ts:%" PRId64 " of generated results out of valid time range %" PRId64 " , discarded", id,
+                  ts, earlyTs);
+          pTableData->aRowP = taosArrayDestroy(pTableData->aRowP);
+          taosArrayDestroy(pVals);
+          return TSDB_CODE_SUCCESS;
+        }
       }
 
       if (IS_SET_NULL(pCol)) {
@@ -605,8 +614,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
           dataIndex++;
         } else {
           void* colData = colDataGetData(pColData, j);
-          if (IS_VAR_DATA_TYPE(pCol->type)) {
-            // address copy, no value
+          if (IS_VAR_DATA_TYPE(pCol->type)) { // address copy, no value
             SValue sv =
                 (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};
             SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
@@ -796,22 +804,25 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
 }
 
 int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
-                               SSubmitTbData* pTableData, const char* id) {
+                               SSubmitTbData* pTableData, int64_t earlyTs, const char* id) {
   int32_t numOfRows = pDataBlock->info.rows;
+  char*   dstTableName = pDataBlock->info.parTbName;
 
   tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, id,
           blockIndex + 1, numOfRows, suid);
-  char* dstTableName = pDataBlock->info.parTbName;
 
   // convert all rows
-  int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, id);
+  int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, earlyTs, id);
   if (code != TSDB_CODE_SUCCESS) {
     tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno));
     return code;
   }
 
-  taosArraySort(pTableData->aRowP, tsAscendingSortFn);
-  tqTrace("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows);
+  if (pTableData->aRowP != NULL) {
+    taosArraySort(pTableData->aRowP, tsAscendingSortFn);
+    tqTrace("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows);
+  }
+
   return code;
 }
 
@@ -836,6 +847,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
   int32_t       numOfBlocks = taosArrayGetSize(pBlocks);
   int32_t       code = TSDB_CODE_SUCCESS;
   const char*   id = pTask->id.idStr;
+  int64_t       earlyTs = tsdbGetEarliestTs(pVnode->pTsdb);
 
   bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks);
   if (!onlySubmitData) {
@@ -870,8 +882,8 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           continue;
         }
 
-        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
-        if (code != TSDB_CODE_SUCCESS) {
+        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
+        if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
           continue;
         }
 
@@ -918,8 +930,12 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           continue;
         }
 
-        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
-        if (code != TSDB_CODE_SUCCESS) {
+        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
+        if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
+          if (tbData.pCreateTbReq != NULL) {
+            tdDestroySVCreateTbReq(tbData.pCreateTbReq);
+            tbData.pCreateTbReq = NULL;
+          }
           continue;
         }
 
@@ -928,8 +944,12 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
         int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
         taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
       } else {
-        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
-        if (code != TSDB_CODE_SUCCESS) {
+        code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
+        if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
+          if (tbData.pCreateTbReq != NULL) {
+            tdDestroySVCreateTbReq(tbData.pCreateTbReq);
+            tbData.pCreateTbReq = NULL;
+          }
           continue;
         }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c
index ea3d285880..f9fede1d9b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbOpen.c
+++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c
@@ -30,6 +30,14 @@ int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) {
   return 0;
 }
 
+int64_t tsdbGetEarliestTs(STsdb *pTsdb) {
+  STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
+
+  int64_t now = taosGetTimestamp(pCfg->precision);
+  int64_t ts = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
+  return ts;
+}
+
 /**
  * @brief
  *
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c
index 99520f7c92..d7c3eff571 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c
@@ -226,14 +226,10 @@ static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey
 // Update the query time window according to the data time to live(TTL) information, in order to avoid to return
 // the expired data to client, even it is queried already.
 static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) {
-  STsdbKeepCfg* pCfg = &pTsdb->keepCfg;
-
-  int64_t now = taosGetTimestamp(pCfg->precision);
-  int64_t earilyTs = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
-
+  int64_t     earlyTs = tsdbGetEarliestTs(pTsdb);
   STimeWindow win = *pWindow;
-  if (win.skey < earilyTs) {
-    win.skey = earilyTs;
+  if (win.skey < earlyTs) {
+    win.skey = earlyTs;
   }
 
   return win;

From b4956392d9cf4768968757d505ec74fd94059daa Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 30 May 2024 15:54:08 +0800
Subject: [PATCH 077/132] fix(stream): return in_progress code if not send
 retrieve rsp to downstream.

---
 source/dnode/vnode/src/tqCommon/tqCommon.c | 7 ++++---
 source/dnode/vnode/src/vnd/vnodeSvr.c      | 8 ++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index ee3f1a3760..45e1a300d0 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -871,7 +871,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
     tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
             " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
             pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId);
-    return TSDB_CODE_SUCCESS;
+    return TSDB_CODE_STREAM_TASK_NOT_EXIST;
   }
 
   tqDebug("s-task:0x%x recv retrieve checkpoint-trigger msg from downstream s-task:0x%x, checkpointId:%" PRId64,
@@ -890,6 +890,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
       tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
               (int32_t)pReq->downstreamTaskId, checkpointId, transId);
       streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info);
+      return TSDB_CODE_SUCCESS;
     } else { // not send checkpoint-trigger yet, wait
       int32_t recv = 0, total = 0;
       streamTaskGetTriggerRecvStatus(pTask, &recv, &total);
@@ -902,6 +903,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
             "sending checkpoint-source/trigger",
             pTask->id.idStr, recv, total);
       }
+      return TSDB_CODE_ACTION_IN_PROGRESS;
     }
   } else {  // upstream not recv the checkpoint-source/trigger till now
     ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT);
@@ -909,9 +911,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
         "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
         "upstream sending checkpoint-source/trigger",
         pTask->id.idStr);
+    return TSDB_CODE_ACTION_IN_PROGRESS;
   }
-
-  return TSDB_CODE_SUCCESS;
 }
 
 int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 426f85fa5e..c0eeebaecb 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -842,16 +842,16 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
       return tqProcessTaskScanHistory(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_CHECKPOINT_READY:
       return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
+    case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
+      return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
     case TDMT_STREAM_RETRIEVE_TRIGGER:
       return tqProcessTaskRetrieveTriggerReq(pVnode->pTq, pMsg);
+    case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:
+      return tqProcessTaskRetrieveTriggerRsp(pVnode->pTq, pMsg);
     case TDMT_MND_STREAM_HEARTBEAT_RSP:
       return tqProcessStreamHbRsp(pVnode->pTq, pMsg);
     case TDMT_MND_STREAM_REQ_CHKPT_RSP:
       return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
-    case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
-      return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
-    case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:
-      return tqProcessTaskRetrieveTriggerRsp(pVnode->pTq, pMsg);
     case TDMT_VND_GET_STREAM_PROGRESS:
       return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
     default:

From d28abdf837b4407ed184db538df33517da87dc07 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 30 May 2024 16:11:53 +0800
Subject: [PATCH 078/132] fix coverity scan problem

---
 include/util/tbuffer.inc                      |  2 +-
 include/util/tutil.h                          |  2 +-
 source/dnode/vnode/src/tsdb/tsdbCommit2.c     | 22 ++++++------
 source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 10 +++---
 source/dnode/vnode/src/tsdb/tsdbFile2.c       |  2 +-
 source/dnode/vnode/src/tsdb/tsdbMerge.c       | 17 +++++----
 source/dnode/vnode/src/tsdb/tsdbSnapInfo.c    |  4 +--
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c    | 35 ++++++++++---------
 source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c |  3 +-
 source/dnode/vnode/src/tsdb/tsdbSttFileRW.c   |  2 +-
 source/dnode/vnode/src/tsdb/tsdbUtil2.c       |  6 ++--
 11 files changed, 58 insertions(+), 47 deletions(-)

diff --git a/include/util/tbuffer.inc b/include/util/tbuffer.inc
index f0137ee978..2c1405d6c5 100644
--- a/include/util/tbuffer.inc
+++ b/include/util/tbuffer.inc
@@ -177,7 +177,7 @@ static int32_t tBufferReaderInit(SBufferReader *reader, uint32_t offset, SBuffer
 }
 
 static FORCE_INLINE int32_t tBufferGet(SBufferReader *reader, uint32_t size, void *data) {
-  if (reader->offset < 0 || reader->offset + size > reader->buffer->size) {
+  if (reader->offset + size > reader->buffer->size) {
     return TSDB_CODE_OUT_OF_RANGE;
   }
   if (data) {
diff --git a/include/util/tutil.h b/include/util/tutil.h
index 54ce6fc849..c049949590 100644
--- a/include/util/tutil.h
+++ b/include/util/tutil.h
@@ -119,7 +119,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
 
 #define TSDB_CHECK_CODE(CODE, LINO, LABEL) \
   do {                                     \
-    if ((CODE)) {                          \
+    if (TSDB_CODE_SUCCESS != (CODE)) {     \
       LINO = __LINE__;                     \
       goto LABEL;                          \
     }                                      \
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index f6082e60e3..183b639550 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -212,6 +212,11 @@ _exit:
   return code;
 }
 
+static int32_t tsdbCommitCloseReader(SCommitter2 *committer) {
+  TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose);
+  return 0;
+}
+
 static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
   int32_t code = 0;
   int32_t lino = 0;
@@ -256,13 +261,17 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
 
 _exit:
   if (code) {
+    tsdbCommitCloseReader(committer);
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   }
   return code;
 }
 
-static int32_t tsdbCommitCloseReader(SCommitter2 *committer) {
-  TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose);
+static int32_t tsdbCommitCloseIter(SCommitter2 *committer) {
+  tsdbIterMergerClose(&committer->tombIterMerger);
+  tsdbIterMergerClose(&committer->dataIterMerger);
+  TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose);
+  TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose);
   return 0;
 }
 
@@ -336,19 +345,12 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
 
 _exit:
   if (code) {
+    tsdbCommitCloseIter(committer);
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   }
   return code;
 }
 
-static int32_t tsdbCommitCloseIter(SCommitter2 *committer) {
-  tsdbIterMergerClose(&committer->tombIterMerger);
-  tsdbIterMergerClose(&committer->dataIterMerger);
-  TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose);
-  TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose);
-  return 0;
-}
-
 static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
   int32_t code = 0;
   int32_t lino = 0;
diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
index acacf98f84..107677ce1e 100644
--- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
@@ -74,7 +74,7 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo
   pBlock->file.stt->level = reader->config->file.stt->level;
 
   int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
-  char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
+  char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
   code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey);
   TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -130,7 +130,7 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil
   TSDB_CHECK_CODE(code, lino, _exit);
 
   int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
-  char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
+  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
 
   if (writer->fd) {
     code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
@@ -211,13 +211,13 @@ _exit:
   return code;
 }
 
-int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock, 
-                                      int32_t encryptAlgorithm, char* encryptKey) {
+int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock,
+                                      int32_t encryptAlgorithm, char *encryptKey) {
   int32_t code = 0;
   int32_t lino = 0;
 
   code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength,
-                        encryptAlgorithm, encryptKey);
+                       encryptAlgorithm, encryptKey);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   writer->ctx->offset += pDataBlock->dataLength;
diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c
index cde74f5e36..7d8b0a6c8b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFile2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c
@@ -44,7 +44,7 @@ static const struct {
 };
 
 void remove_file(const char *fname) {
-  taosRemoveFile(fname);
+  (void)taosRemoveFile(fname);
   tsdbInfo("file:%s is removed", fname);
 }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c
index 022698b0eb..f2d4ab2cb2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMerge.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c
@@ -90,6 +90,11 @@ _exit:
   return code;
 }
 
+static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) {
+  TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose);
+  return 0;
+}
+
 static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
   int32_t  code = 0;
   int32_t  lino = 0;
@@ -220,8 +225,10 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
         code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader);
         TSDB_CHECK_CODE(code, lino, _exit);
 
-        code = TARRAY2_APPEND(merger->sttReaderArr, reader);
-        TSDB_CHECK_CODE(code, lino, _exit);
+        if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) {
+          tsdbSttFileReaderClose(&reader);
+          TSDB_CHECK_CODE(code, lino, _exit);
+        }
       }
     }
 
@@ -232,6 +239,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
 
 _exit:
   if (code) {
+    tsdbMergeFileSetEndCloseReader(merger);
     TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code);
   }
   return code;
@@ -375,11 +383,6 @@ static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) {
   return 0;
 }
 
-static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) {
-  TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose);
-  return 0;
-}
-
 static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
   int32_t code = 0;
   int32_t lino = 0;
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c
index 9dae9bdd36..bc7bf5449e 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c
@@ -321,10 +321,10 @@ int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartLis
         if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
         if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
         if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
-        TARRAY2_APPEND(iList, r);
+        if (TARRAY2_APPEND(iList, r)) goto _err;
       }
     }
-    TARRAY2_APPEND(pList, p);
+    if (TARRAY2_APPEND(pList, p)) goto _err;
     p = NULL;
   }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 0d36f9648b..df89746c31 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -57,6 +57,20 @@ struct STsdbSnapReader {
   STombBlock tombBlock[1];
 };
 
+static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose);
+  tsdbDataFileReaderClose(&reader->dataReader);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
 static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
   int32_t code = 0;
   int32_t lino = 0;
@@ -100,27 +114,16 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
       code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader);
       TSDB_CHECK_CODE(code, lino, _exit);
 
-      code = TARRAY2_APPEND(reader->sttReaderArr, sttReader);
-      TSDB_CHECK_CODE(code, lino, _exit);
+      if ((code = TARRAY2_APPEND(reader->sttReaderArr, sttReader))) {
+        tsdbSttFileReaderClose(&sttReader);
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
     }
   }
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
-  }
-  return code;
-}
-
-static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose);
-  tsdbDataFileReaderClose(&reader->dataReader);
-
-_exit:
-  if (code) {
+    tsdbSnapReadFileSetCloseReader(reader);
     TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
   }
   return code;
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
index ab0575cb53..e4f48745e7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
@@ -141,6 +141,7 @@ static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) {
 
 _exit:
   if (code) {
+    tsdbSnapRAWReadFileSetCloseReader(reader);
     TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
   }
   return code;
@@ -546,7 +547,7 @@ static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbD
   int32_t lino = 0;
 
   int32_t encryptAlgorithm = writer->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
-  char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey;
+  char*   encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey;
 
   code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr, encryptAlgorithm, encryptKey);
   TSDB_CHECK_CODE(code, lino, _exit);
diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
index 478778319d..f1a087e3c1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
@@ -896,7 +896,7 @@ static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) {
   char fname[TSDB_FILENAME_LEN];
   tsdbTFileName(writer->config->tsdb, writer->file, fname);
   tsdbCloseFile(&writer->fd);
-  taosRemoveFile(fname);
+  (void)taosRemoveFile(fname);
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.c b/source/dnode/vnode/src/tsdb/tsdbUtil2.c
index 6df0528f1c..9b88500ad1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.c
@@ -56,7 +56,8 @@ int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record) {
 
   for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
     SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * idx, &tombBlock->buffers[i]);
-    tBufferGetI64(&br, &record->data[i]);
+    int32_t       code = tBufferGetI64(&br, &record->data[i]);
+    if (code) return code;
   }
   return 0;
 }
@@ -182,7 +183,8 @@ int32_t tStatisBlockPut(STbStatisBlock *block, SRowInfo *row, int32_t maxRecords
   if (block->numOfRecords > 0) {
     int64_t       lastUid;
     SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * (block->numOfRecords - 1), &block->uids);
-    tBufferGetI64(&br, &lastUid);
+    int32_t       code = tBufferGetI64(&br, &lastUid);
+    if (code) return code;
 
     if (lastUid == row->uid) {
       return tStatisBlockUpdate(block, row);

From b9d8f63e859c4c238c5cbe79438f6026d981a3f8 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 30 May 2024 16:28:51 +0800
Subject: [PATCH 079/132] fix: memory leak

---
 source/common/src/tmsg.c | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index ef37a41fcf..de24e7047b 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -5616,18 +5616,35 @@ int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *
 }
 
 int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
+  int32_t  code = 0;
   SDecoder decoder = {0};
+
   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+  if (tStartDecode(&decoder) < 0) {
+    code = TSDB_CODE_MSG_DECODE_ERROR;
+    goto _exit;
+  }
 
-  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) {
+    code = TSDB_CODE_MSG_DECODE_ERROR;
+    goto _exit;
+  }
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) {
+    code = TSDB_CODE_MSG_DECODE_ERROR;
+    goto _exit;
+  }
+
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) {
+    code = TSDB_CODE_MSG_DECODE_ERROR;
+    goto _exit;
+  }
 
   tEndDecode(&decoder);
+
+_exit:
   tDecoderClear(&decoder);
-  return 0;
+  return code;
 }
 
 int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeConfigReq *pReq) {

From 1821f9aae402577dda7891a3132dc29a89320343 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 30 May 2024 16:29:31 +0800
Subject: [PATCH 080/132] format

---
 source/common/src/tmsg.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index de24e7047b..c6cf5d1037 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -5630,6 +5630,7 @@ int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq
     code = TSDB_CODE_MSG_DECODE_ERROR;
     goto _exit;
   }
+
   if (tDecodeI32(&decoder, &pReq->vgId) < 0) {
     code = TSDB_CODE_MSG_DECODE_ERROR;
     goto _exit;

From e66283d99ccbefe442f05630b1c164d19062a557 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 30 May 2024 18:41:17 +0800
Subject: [PATCH 081/132] fix(stream): remove table meta in table-meta cache.

---
 source/dnode/vnode/src/tq/tqSink.c | 153 ++++++++++++++++-------------
 1 file changed, 85 insertions(+), 68 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 3fe124ea85..96ccf18504 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -39,6 +39,7 @@ static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STab
                                         const char* dstTableName, int64_t* uid);
 static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId,
                               const char* id);
+static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char* id);
 static bool    isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
 static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName,
                                   int32_t numOfTags);
@@ -396,46 +397,6 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
-                                int64_t suid) {
-  SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))};
-
-  int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr,
-                                  pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
-  }
-
-  if (taosArrayGetSize(deleteReq.deleteReqs) == 0) {
-    taosArrayDestroy(deleteReq.deleteReqs);
-    return TSDB_CODE_SUCCESS;
-  }
-
-  int32_t len;
-  tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code);
-  if (code != TSDB_CODE_SUCCESS) {
-    qError("s-task:%s failed to encode delete request", pTask->id.idStr);
-    return code;
-  }
-
-  SEncoder encoder;
-  void*    serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
-  void*    abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
-  tEncoderInit(&encoder, abuf, len);
-  tEncodeSBatchDeleteReq(&encoder, &deleteReq);
-  tEncoderClear(&encoder);
-  taosArrayDestroy(deleteReq.deleteReqs);
-
-  ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode);
-
-  SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)};
-  if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
-    tqDebug("failed to put delete req into write-queue since %s", terrstr());
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
 bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid) {
   if (pReader->me.type != TSDB_CHILD_TABLE) {
     tqError("vgId:%d, failed to write into %s, since table type:%d incorrect", vgId, ctbName, pReader->me.type);
@@ -484,23 +445,6 @@ SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, in
   return pCreateTbReq;
 }
 
-int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) {
-  if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) {
-    taosMemoryFreeClear(pTableSinkInfo);  // too many items, failed to cache it
-    return TSDB_CODE_FAILED;
-  }
-
-  int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES);
-  if (code != TSDB_CODE_SUCCESS) {
-    taosMemoryFreeClear(pTableSinkInfo);
-  } else {
-    tqDebug("s-task:%s new dst table:%s(uid:%" PRIu64 ") added into cache, total:%d", id, pTableSinkInfo->name.data,
-            pTableSinkInfo->uid, tSimpleHashGetSize(pSinkTableMap));
-  }
-
-  return code;
-}
-
 int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) {
   int32_t code = 0;
   void*   pBuf = NULL;
@@ -826,17 +770,6 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_
   return code;
 }
 
-bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) {
-  for (int32_t i = 0; i < numOfBlocks; ++i) {
-    SSDataBlock* p = taosArrayGet(pBlocks, i);
-    if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
 void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
   const SArray* pBlocks = (const SArray*)data;
   SVnode*       pVnode = (SVnode*)vnode;
@@ -884,6 +817,11 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
 
         code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
         if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
+          if (tbData.pCreateTbReq != NULL) {
+            tdDestroySVCreateTbReq(tbData.pCreateTbReq);
+            doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id);
+            tbData.pCreateTbReq = NULL;
+          }
           continue;
         }
 
@@ -934,6 +872,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
         if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
           if (tbData.pCreateTbReq != NULL) {
             tdDestroySVCreateTbReq(tbData.pCreateTbReq);
+            doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id);
             tbData.pCreateTbReq = NULL;
           }
           continue;
@@ -973,3 +912,81 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
     }
   }
 }
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) {
+  for (int32_t i = 0; i < numOfBlocks; ++i) {
+    SSDataBlock* p = taosArrayGet(pBlocks, i);
+    if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) {
+  if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) {
+    taosMemoryFreeClear(pTableSinkInfo);  // too many items, failed to cache it
+    return TSDB_CODE_FAILED;
+  }
+
+  int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES);
+  if (code != TSDB_CODE_SUCCESS) {
+    taosMemoryFreeClear(pTableSinkInfo);
+  } else {
+    tqDebug("s-task:%s new dst table:%s(uid:%" PRIu64 ") added into cache, total:%d", id, pTableSinkInfo->name.data,
+            pTableSinkInfo->uid, tSimpleHashGetSize(pSinkTableMap));
+  }
+
+  return code;
+}
+
+int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char* id) {
+  if (tSimpleHashGetSize(pSinkTableMap) == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId));
+  tqDebug("s-task:%s remove cached table meta for groupId:%"PRId64, id, groupId);
+}
+
+int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
+                                int64_t suid) {
+  SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))};
+
+  int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr,
+                                  pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  if (taosArrayGetSize(deleteReq.deleteReqs) == 0) {
+    taosArrayDestroy(deleteReq.deleteReqs);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t len;
+  tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code);
+  if (code != TSDB_CODE_SUCCESS) {
+    qError("s-task:%s failed to encode delete request", pTask->id.idStr);
+    return code;
+  }
+
+  SEncoder encoder;
+  void*    serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
+  void*    abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
+  tEncoderInit(&encoder, abuf, len);
+  tEncodeSBatchDeleteReq(&encoder, &deleteReq);
+  tEncoderClear(&encoder);
+  taosArrayDestroy(deleteReq.deleteReqs);
+
+  ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode);
+
+  SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)};
+  if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+    tqDebug("failed to put delete req into write-queue since %s", terrstr());
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
\ No newline at end of file

From 815a9e953ba31eb84f6495bc60fd7aa73d09b49d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 30 May 2024 18:41:56 +0800
Subject: [PATCH 082/132] fix(stream): desc the ref for checkpoint-trigger
 timer.

---
 source/dnode/vnode/src/tq/tqSink.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 96ccf18504..5f3e1e3d14 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -948,7 +948,8 @@ int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char
   }
 
   int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId));
-  tqDebug("s-task:%s remove cached table meta for groupId:%"PRId64, id, groupId);
+  tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId);
+  return code;
 }
 
 int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,

From c6407277dc27e92dee64af1abb6524d482991b23 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Thu, 30 May 2024 11:41:22 +0000
Subject: [PATCH 083/132] support add column compress

---
 include/common/tmsg.h                   | 77 ++++++++++-----------
 source/common/src/tmsg.c                | 61 ++++++++++++++---
 source/dnode/mnode/impl/src/mndStb.c    | 76 ++++++++++++++-------
 source/dnode/vnode/src/meta/metaTable.c | 20 ++++--
 source/libs/parser/inc/parAst.h         | 90 ++++++++++++++-----------
 source/libs/parser/inc/sql.y            |  2 +-
 source/libs/parser/src/parAstCreater.c  | 29 ++++++--
 source/libs/parser/src/parTranslater.c  | 62 +++++++++++++----
 8 files changed, 282 insertions(+), 135 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index be2f443140..3c8f2be059 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -158,19 +158,20 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_MAX,
 } EShowType;
 
-#define TSDB_ALTER_TABLE_ADD_TAG                1
-#define TSDB_ALTER_TABLE_DROP_TAG               2
-#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME        3
-#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL         4
-#define TSDB_ALTER_TABLE_ADD_COLUMN             5
-#define TSDB_ALTER_TABLE_DROP_COLUMN            6
-#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES    7
-#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES       8
-#define TSDB_ALTER_TABLE_UPDATE_OPTIONS         9
-#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME     10
-#define TSDB_ALTER_TABLE_ADD_TAG_INDEX          11
-#define TSDB_ALTER_TABLE_DROP_TAG_INDEX         12
-#define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS 13
+#define TSDB_ALTER_TABLE_ADD_TAG                         1
+#define TSDB_ALTER_TABLE_DROP_TAG                        2
+#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME                 3
+#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL                  4
+#define TSDB_ALTER_TABLE_ADD_COLUMN                      5
+#define TSDB_ALTER_TABLE_DROP_COLUMN                     6
+#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES             7
+#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES                8
+#define TSDB_ALTER_TABLE_UPDATE_OPTIONS                  9
+#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME              10
+#define TSDB_ALTER_TABLE_ADD_TAG_INDEX                   11
+#define TSDB_ALTER_TABLE_DROP_TAG_INDEX                  12
+#define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS          13
+#define TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION 14
 
 #define TSDB_FILL_NONE        0
 #define TSDB_FILL_NULL        1
@@ -2313,10 +2314,10 @@ int32_t tDeserializeSVArbHeartBeatRsp(void* buf, int32_t bufLen, SVArbHeartBeatR
 void    tFreeSVArbHeartBeatRsp(SVArbHeartBeatRsp* pRsp);
 
 typedef struct {
-  char* arbToken;
+  char*   arbToken;
   int64_t arbTerm;
-  char* member0Token;
-  char* member1Token;
+  char*   member0Token;
+  char*   member1Token;
 } SVArbCheckSyncReq;
 
 int32_t tSerializeSVArbCheckSyncReq(void* buf, int32_t bufLen, SVArbCheckSyncReq* pReq);
@@ -2336,9 +2337,9 @@ int32_t tDeserializeSVArbCheckSyncRsp(void* buf, int32_t bufLen, SVArbCheckSyncR
 void    tFreeSVArbCheckSyncRsp(SVArbCheckSyncRsp* pRsp);
 
 typedef struct {
-  char*    arbToken;
-  int64_t  arbTerm;
-  char*    memberToken;
+  char*   arbToken;
+  int64_t arbTerm;
+  char*   memberToken;
 } SVArbSetAssignedLeaderReq;
 
 int32_t tSerializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq);
@@ -2700,8 +2701,8 @@ typedef struct {
   int64_t  lastTs;
   SArray*  pVgroupVerList;
   // 3.3.0.0
-  SArray*  pCols;  // array of SField
-  int64_t  smaId;
+  SArray* pCols;  // array of SField
+  int64_t smaId;
 } SCMCreateStreamReq;
 
 typedef struct {
@@ -3569,7 +3570,7 @@ static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t u
   pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA;
   pOffsetVal->uid = uid;
   pOffsetVal->ts = ts;
-  if(IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)){
+  if (IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)) {
     taosMemoryFree(pOffsetVal->primaryKey.pData);
   }
   pOffsetVal->primaryKey = primaryKey;
@@ -3669,10 +3670,10 @@ typedef struct {
   char*   ast;
   int64_t deleteMark;
   int64_t lastTs;
-  int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0
+  int64_t normSourceTbUid;  // the Uid of source tb if its a normal table, otherwise 0
   SArray* pVgroupVerList;
   int8_t  recursiveTsma;
-  char    baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma
+  char    baseTsmaName[TSDB_TABLE_FNAME_LEN];  // base tsma name for recursively created tsma
 } SMCreateSmaReq;
 
 int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq);
@@ -3901,7 +3902,7 @@ typedef struct {
 
 int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
 int32_t tDeserializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
-void    tDestroySMqPollReq(SMqPollReq *pReq);
+void    tDestroySMqPollReq(SMqPollReq* pReq);
 
 typedef struct {
   int32_t vgId;
@@ -3945,7 +3946,7 @@ typedef struct {
 
 int32_t tEncodeMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp);
 int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp);
-void    tDeleteMqMetaRsp(SMqMetaRsp *pRsp);
+void    tDeleteMqMetaRsp(SMqMetaRsp* pRsp);
 
 #define MQ_DATA_RSP_VERSION 100
 
@@ -4292,9 +4293,9 @@ int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pR
 int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
 void    tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
 typedef struct {
-  char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name
-  bool fetchingWithTsmaName; // if we are fetching with tsma name
-}STableTSMAInfoReq;
+  char name[TSDB_TABLE_FNAME_LEN];  // table name or tsma name
+  bool fetchingWithTsmaName;        // if we are fetching with tsma name
+} STableTSMAInfoReq;
 
 int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq);
 int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq);
@@ -4317,15 +4318,15 @@ typedef struct {
   int32_t  version;
   int64_t  interval;
   int8_t   unit;
-  SArray*  pFuncs;  // SArray<STableTSMAFuncInfo>
-  SArray*  pTags; // SArray<SSchema>
-  SArray*  pUsedCols; // SArray<SSchema>
+  SArray*  pFuncs;     // SArray<STableTSMAFuncInfo>
+  SArray*  pTags;      // SArray<SSchema>
+  SArray*  pUsedCols;  // SArray<SSchema>
   char*    ast;
 
   int64_t streamUid;
   int64_t reqTs;
   int64_t rspTs;
-  int64_t delayDuration; // ms
+  int64_t delayDuration;  // ms
   bool    fillHistoryFinished;
 } STableTSMAInfo;
 
@@ -4336,10 +4337,10 @@ void    tFreeTableTSMAInfo(void* p);
 void    tFreeAndClearTableTSMAInfo(void* p);
 void    tFreeTableTSMAInfoRsp(STableTSMAInfoRsp* pRsp);
 
-#define STSMAHbRsp STableTSMAInfoRsp
-#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp
+#define STSMAHbRsp            STableTSMAInfoRsp
+#define tSerializeTSMAHbRsp   tSerializeTableTSMAInfoRsp
 #define tDeserializeTSMAHbRsp tDeserializeTableTSMAInfoRsp
-#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp
+#define tFreeTSMAHbRsp        tFreeTableTSMAInfoRsp
 
 typedef struct SStreamProgressReq {
   int64_t streamId;
@@ -4365,7 +4366,7 @@ int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgres
 
 typedef struct SDropCtbWithTsmaSingleVgReq {
   SVgroupInfo vgInfo;
-  SArray*     pTbs; // SVDropTbReq
+  SArray*     pTbs;  // SVDropTbReq
 } SMDropTbReqsOnSingleVg;
 
 int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder* pEncoder, const SMDropTbReqsOnSingleVg* pReq);
@@ -4373,7 +4374,7 @@ int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg*
 void    tFreeSMDropTbReqOnSingleVg(void* p);
 
 typedef struct SDropTbsReq {
-  SArray* pVgReqs; // SMDropTbReqsOnSingleVg
+  SArray* pVgReqs;  // SMDropTbReqsOnSingleVg
 } SMDropTbsReq;
 
 int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index ef37a41fcf..624866e396 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -768,11 +768,22 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1;
+
+  // if (pReq->alterType == )
   for (int32_t i = 0; i < pReq->numOfFields; ++i) {
-    SField *pField = taosArrayGet(pReq->pFields, i);
-    if (tEncodeI8(&encoder, pField->type) < 0) return -1;
-    if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
-    if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
+    if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
+      SFieldWithOptions *pField = taosArrayGet(pReq->pFields, i);
+      if (tEncodeI8(&encoder, pField->type) < 0) return -1;
+      if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
+      if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
+      if (tEncodeU32(&encoder, pField->compress) < 0) return -1;
+
+    } else {
+      SField *pField = taosArrayGet(pReq->pFields, i);
+      if (tEncodeI8(&encoder, pField->type) < 0) return -1;
+      if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
+      if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
+    }
   }
   if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
@@ -802,13 +813,28 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
   }
 
   for (int32_t i = 0; i < pReq->numOfFields; ++i) {
-    SField field = {0};
-    if (tDecodeI8(&decoder, &field.type) < 0) return -1;
-    if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
-    if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
-    if (taosArrayPush(pReq->pFields, &field) == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
+    if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
+
+      taosArrayDestroy(pReq->pFields);
+      pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SFieldWithOptions));
+      SFieldWithOptions field = {0};
+      if (tDecodeI8(&decoder, &field.type) < 0) return -1;
+      if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
+      if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
+      if (tDecodeU32(&decoder, &field.compress) < 0) return -1;
+      if (taosArrayPush(pReq->pFields, &field) == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
+    } else {
+      SField field = {0};
+      if (tDecodeI8(&decoder, &field.type) < 0) return -1;
+      if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
+      if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
+      if (taosArrayPush(pReq->pFields, &field) == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
     }
   }
 
@@ -8740,6 +8766,13 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
       if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
       if (tEncodeU32(pEncoder, pReq->compress) < 0) return -1;
       break;
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
+      if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->type) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->flags) < 0) return -1;
+      if (tEncodeI32v(pEncoder, pReq->bytes) < 0) return -1;
+      if (tEncodeU32(pEncoder, pReq->compress) < 0) return -1;
+      break;
     default:
       break;
   }
@@ -8795,6 +8828,12 @@ static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq)
       if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
       if (tDecodeU32(pDecoder, &pReq->compress) < 0) return -1;
       break;
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
+      if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->type) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->flags) < 0) return -1;
+      if (tDecodeI32v(pDecoder, &pReq->bytes) < 0) return -1;
+      if (tDecodeU32(pDecoder, &pReq->compress) < 0) return -1;
     default:
       break;
   }
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 61fc180dc6..1ac9200357 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -1779,7 +1779,8 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
 
   return 0;
 }
-static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols) {
+static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols,
+                                      int8_t withCompress) {
   if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) {
     terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS;
     return -1;
@@ -1806,29 +1807,53 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
   }
 
   for (int32_t i = 0; i < ncols; i++) {
-    SField *pField = taosArrayGet(pFields, i);
-    if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
-      terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
-      return -1;
+    if (withCompress) {
+      SFieldWithOptions *pField = taosArrayGet(pFields, i);
+      if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
+        terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
+        return -1;
+      }
+
+      if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
+        terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
+        return -1;
+      }
+
+      SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i];
+      pSchema->bytes = pField->bytes;
+      pSchema->type = pField->type;
+      memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
+      pSchema->colId = pNew->nextColId;
+      pNew->nextColId++;
+
+      SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
+      pCmpr->id = pSchema->colId;
+      pCmpr->alg = createDefaultColCmprByType(pSchema->type);
+      mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
+    } else {
+      SField *pField = taosArrayGet(pFields, i);
+      if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
+        terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
+        return -1;
+      }
+
+      if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
+        terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
+        return -1;
+      }
+
+      SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i];
+      pSchema->bytes = pField->bytes;
+      pSchema->type = pField->type;
+      memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
+      pSchema->colId = pNew->nextColId;
+      pNew->nextColId++;
+
+      SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
+      pCmpr->id = pSchema->colId;
+      pCmpr->alg = createDefaultColCmprByType(pSchema->type);
+      mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
     }
-
-    if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) {
-      terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
-      return -1;
-    }
-
-    SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i];
-    pSchema->bytes = pField->bytes;
-    pSchema->type = pField->type;
-    memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
-    pSchema->colId = pNew->nextColId;
-    pNew->nextColId++;
-
-    SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
-    pCmpr->id = pSchema->colId;
-    pCmpr->alg = createDefaultColCmprByType(pSchema->type);
-
-    mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
   }
 
   pNew->colVer++;
@@ -2461,7 +2486,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
       code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0);
       break;
     case TSDB_ALTER_TABLE_ADD_COLUMN:
-      code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
+      code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 0);
       break;
     case TSDB_ALTER_TABLE_DROP_COLUMN:
       pField0 = taosArrayGet(pAlter->pFields, 0);
@@ -2478,6 +2503,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS:
       code = mndUpdateSuperTableColumnCompress(pMnode, pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
       break;
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
+      code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 1);
+      break;
     default:
       needRsp = false;
       terrno = TSDB_CODE_OPS_NOT_SUPPORT;
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index df54591ae8..e8ce39f9fb 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -1491,6 +1491,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
   SSchema  tScheam;
   switch (pAlterTbReq->action) {
     case TSDB_ALTER_TABLE_ADD_COLUMN:
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
       if (pColumn) {
         terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS;
         goto _err;
@@ -1522,10 +1523,21 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         (void)tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type);
       }
       SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1];
-      updataTableColCmpr(&entry.colCmpr, pCol, 1);
-      freeColCmpr = true;
-      ASSERT(entry.colCmpr.nCols == pSchema->nCols);
-      break;
+      if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
+        
+        // if (pAlterTbReq->colCmpr.nCols != pSchema->nCols) {
+        //   terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
+        //   goto _err;
+        // }
+        // updataTableColCmpr(&entry.colCmpr, pCol, 1);
+        // freeColCmpr = true;
+        // ASSERT(entry.colCmpr.nCols == pSchema->nCols);
+      } else {
+        updataTableColCmpr(&entry.colCmpr, pCol, 1);
+        freeColCmpr = true;
+        ASSERT(entry.colCmpr.nCols == pSchema->nCols);
+        break;
+      }
     case TSDB_ALTER_TABLE_DROP_COLUMN:
       if (pColumn == NULL) {
         terrno = TSDB_CODE_VND_COL_NOT_EXISTS;
diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h
index e8596c4d19..90c3753ed0 100644
--- a/source/libs/parser/inc/parAst.h
+++ b/source/libs/parser/inc/parAst.h
@@ -118,44 +118,46 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode);
 SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode);
 SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode);
 
-SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName);
-SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral);
-SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode);
-SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight);
+SNode*     createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName);
+SNode*     createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral);
+SNode*     createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pNode);
+SNode*     createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pLeft,
+                                 SNode* pRight);
 SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
-SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
-SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
-SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
-SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
-SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
-SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
-SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
-SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
-SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
-SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
-SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
-SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
-SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
-SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
-SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
-SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
-SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
-SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
-SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
-SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
-SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
-SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
-SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
-SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
-SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
-                                SNode* pFill);
-SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);                                
-SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
-SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
-SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
-SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
-SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
-SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
+SNode*     createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
+SNode*     createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
+SNode*     createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
+SNode*     createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
+SNode*     createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
+SNode*     setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
+SNode*     createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
+SNode*     createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
+SNode*     createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
+SNode*     createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
+SNode*     createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
+SNode*     createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
+SNode*     createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
+SNode*     createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
+SNode*     createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
+SNode*     createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
+SNode*     createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight,
+                               SNode* pJoinCond);
+SNode*     createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
+SNode*     createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
+SNode*     createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
+SNode*     createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
+SNode*     createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
+SNode*     createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
+SNode*     createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
+SNode*     createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
+                                    SNode* pFill);
+SNode*     createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);
+SNode*     createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
+SNode*     createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
+SNode*     createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
+SNode*     createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
+SNode*     createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
+SNode*     createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
 
 SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
 SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);
@@ -170,7 +172,8 @@ SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
 SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
 SNode* addJLimitClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pJLimit);
 SNode* addWindowOffsetClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pWinOffset);
-SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
+SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable,
+                        SNodeList* pHint);
 SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
 SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
 
@@ -205,8 +208,12 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S
 SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions);
 SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName,
                                     SDataType dataType);
-SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName,
-                                    SNode* pOptions);
+
+SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
+                                            SToken* pColName, SDataType dataType, SNode* pOptions);
+
+SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
+                                           SToken* pColName, SNode* pOptions);
 SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName);
 SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pOldColName,
                                  SToken* pNewColName);
@@ -217,7 +224,8 @@ SNode* setShowKind(SAstCreateContext* pCxt, SNode* pStmt, EShowKind showKind);
 SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type);
 SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName,
                               EOperatorType tableCondType);
-SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType);
+SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName,
+                            EOperatorType tableCondType);
 SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
 SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type);
 SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y
index 60e33c6f33..a5efdbdf91 100644
--- a/source/libs/parser/inc/sql.y
+++ b/source/libs/parser/inc/sql.y
@@ -353,7 +353,7 @@ cmd ::= ALTER STABLE alter_table_clause(A).
 
 alter_table_clause(A) ::= full_table_name(B) alter_table_options(C).              { A = createAlterTableModifyOptions(pCxt, B, C); }
 alter_table_clause(A) ::=
-  full_table_name(B) ADD COLUMN column_name(C) type_name(D).                      { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); }
+  full_table_name(B) ADD COLUMN column_name(C) type_name(D) column_options(E).                      { A = createAlterTableAddModifyColOptions2(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D, E); }
 alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C).          { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); }
 alter_table_clause(A) ::=
   full_table_name(B) MODIFY COLUMN column_name(C) type_name(D).                   { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); }
diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 5399aa7f09..86ce93d920 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -176,7 +176,8 @@ static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb)
 
 static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
   trimEscape(pTableName);
-  if (NULL != pTableName && pTableName->type != TK_NK_NIL && (pTableName->n >= TSDB_TABLE_NAME_LEN || pTableName->n == 0)) {
+  if (NULL != pTableName && pTableName->type != TK_NK_NIL &&
+      (pTableName->n >= TSDB_TABLE_NAME_LEN || pTableName->n == 0)) {
     pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z);
     return false;
   }
@@ -185,7 +186,8 @@ static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
 
 static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) {
   trimEscape(pColumnName);
-  if (NULL != pColumnName && pColumnName->type != TK_NK_NIL && (pColumnName->n >= TSDB_COL_NAME_LEN || pColumnName->n == 0)) {
+  if (NULL != pColumnName && pColumnName->type != TK_NK_NIL &&
+      (pColumnName->n >= TSDB_COL_NAME_LEN || pColumnName->n == 0)) {
     pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z);
     return false;
   }
@@ -1893,6 +1895,24 @@ SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable,
   pStmt->dataType = dataType;
   return createAlterTableStmtFinalize(pRealTable, pStmt);
 }
+SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
+                                            SToken* pColName, SDataType dataType, SNode* pOptions) {
+  CHECK_PARSER_STATUS(pCxt);
+  if (!checkColumnName(pCxt, pColName)) {
+    return NULL;
+  }
+  SAlterTableStmt* pStmt = (SAlterTableStmt*)nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  pStmt->alterType = alterType;
+  COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName);
+  pStmt->dataType = dataType;
+
+  if (pOptions != NULL) {
+    pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION;
+  }
+  pStmt->pColOptions = (SColumnOptions*)pOptions;
+  return createAlterTableStmtFinalize(pRealTable, pStmt);
+}
 
 SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType,
                                            SToken* pColName, SNode* pOptions) {
@@ -2816,9 +2836,10 @@ SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgI
   return (SNode*)pStmt;
 }
 
-SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName){
+SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
   CHECK_PARSER_STATUS(pCxt);
-  SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT);
+  SBalanceVgroupLeaderStmt* pStmt =
+      (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   if (NULL != pDbName) {
     COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index c9a05a348c..938c6aae9a 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1003,7 +1003,7 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) {
 
 SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) {
   SNodeList* pPartitionList = NULL;
-  SNode* pNode = NULL;
+  SNode*     pNode = NULL;
   if (pOrderList->length <= nodesNum) {
     return NULL;
   }
@@ -1023,7 +1023,6 @@ SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesN
   return pPartitionList;
 }
 
-
 static bool isTimeLineAlignedQuery(SNode* pStmt) {
   SSelectStmt* pSelect = (SSelectStmt*)pStmt;
   if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
@@ -1036,7 +1035,8 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
         return true;
       }
       if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
-        SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
+        SNodeList* pPartitionList =
+            buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
         bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
         nodesDestroyList(pPartitionList);
 
@@ -1049,7 +1049,8 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
   if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
     SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
     if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
-      SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
+      SNodeList* pPartitionList =
+          buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
       bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
       nodesDestroyList(pPartitionList);
 
@@ -6083,7 +6084,7 @@ static void resetResultTimeline(SSelectStmt* pSelect) {
       }
     }
   }
-  
+
   pSelect->timeLineResMode = TIME_LINE_NONE;
 }
 
@@ -6237,8 +6238,8 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
     }
     snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
     SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
-    bool isLeftPrimTs = isPrimaryKeyImpl(pLeft);
-    bool isRightPrimTs = isPrimaryKeyImpl(pRight);
+    bool   isLeftPrimTs = isPrimaryKeyImpl(pLeft);
+    bool   isRightPrimTs = isPrimaryKeyImpl(pRight);
 
     if (isLeftPrimTs && isRightPrimTs) {
       SColumnNode* pFCol = (SColumnNode*)pProj;
@@ -6288,9 +6289,9 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
           pSetOperator->timeLineFromOrderBy = true;
           return code;
         }
-      
}
+      }
     }
-    
+
     pSetOperator->timeLineResMode = TIME_LINE_NONE;
   }
   return code;
@@ -8156,9 +8157,9 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
   }
 
   switch (pStmt->alterType) {
+    case TSDB_ALTER_TABLE_ADD_COLUMN:
     case TSDB_ALTER_TABLE_ADD_TAG:
     case TSDB_ALTER_TABLE_DROP_TAG:
-    case TSDB_ALTER_TABLE_ADD_COLUMN:
     case TSDB_ALTER_TABLE_DROP_COLUMN:
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
     case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: {
@@ -8193,6 +8194,28 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
       taosArrayPush(pAlterReq->pFields, &field);
       break;
     }
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
+      taosArrayDestroy(pAlterReq->pFields);
+
+      pAlterReq->pFields = taosArrayInit(1, sizeof(SFieldWithOptions));
+      SFieldWithOptions field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)};
+      // TAOS_FIELD        field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)};
+      strcpy(field.name, pStmt->colName);
+      if (pStmt->pColOptions != NULL) {
+        if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
+        if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
+        if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
+        int32_t code = setColCompressByOption(pStmt->dataType.type, columnEncodeVal(pStmt->pColOptions->encode),
+                                              columnCompressVal(pStmt->pColOptions->compress),
+                                              columnLevelVal(pStmt->pColOptions->compressLevel), false,
+                                              (uint32_t*)&field.compress);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
+      }
+      taosArrayPush(pAlterReq->pFields, &field);
+      break;
+    }
     default:
       break;
   }
@@ -10642,7 +10665,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
 static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
   SBalanceVgroupLeaderReq req = {0};
   req.vgId = pStmt->vgId;
-  if(pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName);
+  if (pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName);
   int32_t code =
       buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
   tFreeSBalanceVgroupLeaderReq(&req);
@@ -11057,7 +11080,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    pReq->deleteMark = convertTimePrecision(tsmaDataDeleteMark, TSDB_TIME_PRECISION_MILLI, pTableMeta->tableInfo.precision);
+    pReq->deleteMark =
+        convertTimePrecision(tsmaDataDeleteMark, TSDB_TIME_PRECISION_MILLI, pTableMeta->tableInfo.precision);
     code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen);
   }
 
@@ -12857,6 +12881,11 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW);
   }
 
+  // only super and normal support
+  if (pStmt->pColOptions != NULL && TSDB_CHILD_TABLE == pTableMeta->tableType) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
+  }
+
   pReq->colName = taosStrdup(pStmt->colName);
   if (NULL == pReq->colName) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -12865,6 +12894,15 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S
   pReq->type = pStmt->dataType.type;
   pReq->flags = COL_SMA_ON;
   pReq->bytes = calcTypeBytes(pStmt->dataType);
+  if (pStmt->pColOptions != NULL) {
+    if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
+    if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
+    if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
+    int8_t code = setColCompressByOption(pReq->type, columnEncodeVal(pStmt->pColOptions->encode),
+                                         columnCompressVal(pStmt->pColOptions->compress),
+                                         columnLevelVal(pStmt->pColOptions->compressLevel), true, &pReq->compress);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 

From ee4e4c0e6d4d75c43637746fff13ce5069b637ec Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 31 May 2024 09:17:20 +0800
Subject: [PATCH 084/132] refactor: do some internal refactor.

---
 include/libs/stream/tstream.h                |  2 +-
 source/dnode/mnode/impl/inc/mndStream.h      | 11 ----------
 source/dnode/mnode/impl/src/mndStream.c      |  8 +++----
 source/dnode/mnode/impl/src/mndStreamTrans.c | 22 --------------------
 source/dnode/mnode/impl/src/mndStreamUtil.c  |  7 -------
 source/dnode/snode/src/snode.c               |  4 ++--
 source/dnode/vnode/src/sma/smaRollup.c       |  4 ++--
 source/dnode/vnode/src/tq/tq.c               |  8 +++----
 source/libs/stream/inc/streamInt.h           |  3 ---
 source/libs/stream/src/streamDispatch.c      | 22 --------------------
 source/libs/stream/src/streamMeta.c          |  8 +++----
 source/libs/stream/src/streamQueue.c         |  2 +-
 source/libs/stream/src/streamSched.c         |  8 +++----
 source/libs/stream/src/streamTask.c          |  2 +-
 source/libs/stream/src/streammsg.c           |  4 ++--
 15 files changed, 24 insertions(+), 91 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 7e1c80c842..d007ea29a9 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -304,7 +304,7 @@ typedef struct SSTaskBasicInfo {
   int32_t totalLevel;
   int8_t  taskLevel;
   int8_t  fillHistory;   // is fill history task or not
-  int64_t triggerParam;  // in msec
+  int64_t delaySchedParam;  // in msec
 } SSTaskBasicInfo;
 
 typedef struct SStreamRetrieveReq SStreamRetrieveReq;
diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index bb10a9b9ad..2800aecdfa 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -46,18 +46,8 @@ typedef struct SVgroupChangeInfo {
   SArray   *pUpdateNodeList;  // SArray<SNodeUpdateInfo>
 } SVgroupChangeInfo;
 
-// time to generated the checkpoint, if now() - checkpointTs >= tsCheckpointInterval, this checkpoint will be discard
-// to avoid too many checkpoints for a taskk in the waiting list
-typedef struct SCheckpointCandEntry {
-  char   *pName;
-  int64_t streamId;
-  int64_t checkpointTs;
-  int64_t checkpointId;
-} SCheckpointCandEntry;
-
 typedef struct SStreamTransMgmt {
   SHashObj *pDBTrans;
-  SHashObj *pWaitingList;  // stream id list, of which timed checkpoint failed to be issued due to the trans conflict.
 } SStreamTransMgmt;
 
 typedef struct SStreamExecInfo {
@@ -98,7 +88,6 @@ int32_t     mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
 int32_t     mndPersistStream(STrans *pTrans, SStreamObj *pStream);
 int32_t     mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId);
 int32_t     mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt);
-int32_t     mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId);
 bool        mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock);
 int32_t     mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId);
 
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 09e0e4e415..b88c4a4665 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -45,7 +45,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq);
 static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq);
 
-static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
+static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq);
 static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextStream(SMnode *pMnode, void *pIter);
 static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
@@ -112,7 +112,7 @@ int32_t mndInitStream(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_STREAM_DROP_RSP, mndTransProcessRsp);
 
   mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
-  mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
+  mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint);
   mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint);
   mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
   mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
@@ -141,7 +141,6 @@ void mndCleanupStream(SMnode *pMnode) {
   taosArrayDestroy(execInfo.pTaskList);
   taosHashCleanup(execInfo.pTaskMap);
   taosHashCleanup(execInfo.transMgmt.pDBTrans);
-  taosHashCleanup(execInfo.transMgmt.pWaitingList);
   taosHashCleanup(execInfo.pTransferStateStreams);
   taosThreadMutexDestroy(&execInfo.lock);
   mDebug("mnd stream exec info cleanup");
@@ -967,7 +966,6 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
 
   bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock);
   if (conflict) {
-    mndAddtoCheckpointWaitingList(pStream, checkpointId);
     mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb,
           pStream->name, pStream->uid);
     return -1;
@@ -1131,7 +1129,7 @@ static int32_t streamWaitComparFn(const void* p1, const void* p2) {
   return pInt1->duration > pInt2->duration? -1:1;
 }
 
-static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
+static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
   SMnode     *pMnode = pReq->info.node;
   SSdb       *pSdb = pMnode->pSdb;
   void       *pIter = NULL;
diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c
index ac4cb08308..ff31aa0f7d 100644
--- a/source/dnode/mnode/impl/src/mndStreamTrans.c
+++ b/source/dnode/mnode/impl/src/mndStreamTrans.c
@@ -151,28 +151,6 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) {
   return 0;
 }
 
-int32_t mndAddtoCheckpointWaitingList(SStreamObj* pStream, int64_t checkpointId) {
-  SCheckpointCandEntry* pEntry = taosHashGet(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid));
-  if (pEntry == NULL) {
-    SCheckpointCandEntry entry = {.streamId = pStream->uid,
-                                  .checkpointTs = taosGetTimestampMs(),
-                                  .checkpointId = checkpointId,
-                                  .pName = taosStrdup(pStream->name)};
-
-    taosHashPut(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid), &entry, sizeof(entry));
-    int32_t size = taosHashGetSize(execInfo.transMgmt.pWaitingList);
-
-    mDebug("stream:%" PRIx64 " add into waiting list due to conflict, ts:%" PRId64 " , checkpointId: %" PRId64
-           ", total in waitingList:%d",
-           pStream->uid, entry.checkpointTs, checkpointId, size);
-  } else {
-    mDebug("stream:%" PRIx64 " ts:%" PRId64 ", checkpointId:%" PRId64 " already in waiting list, no need to add into",
-           pStream->uid, pEntry->checkpointTs, checkpointId);
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
 STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg) {
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
   if (pTrans == NULL) {
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index d5bc12f9df..e53908eeed 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -558,11 +558,6 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *
   return 0;
 }
 
-static void freeCheckpointCandEntry(void *param) {
-  SCheckpointCandEntry *pEntry = param;
-  taosMemoryFreeClear(pEntry->pName);
-}
-
 static void freeTaskList(void* param) {
   SArray** pList = (SArray **)param;
   taosArrayDestroy(*pList);
@@ -575,9 +570,7 @@ void mndInitExecInfo() {
   execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
   execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK);
   execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK);
-  execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK);
   execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK);
 
-  taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry);
   taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
 }
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 89ab6d52c3..c61988574c 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -62,14 +62,14 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
             " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
             SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
             pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
-            (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
+            (int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam);
   } else {
     sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
             " nextProcessVer:%" PRId64
             " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
             SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
             pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
-            (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
+            (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam);
   }
   return 0;
 }
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 7138ecbeaa..3cc7c6ec66 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -244,7 +244,7 @@ static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTask
   SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
   if (ppTask && *ppTask) {
     pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer;
-    pItem->fetchResultVer = (*ppTask)->info.triggerParam;
+    pItem->fetchResultVer = (*ppTask)->info.delaySchedParam;
   }
   streamMetaRUnLock(pMeta);
 }
@@ -1289,7 +1289,7 @@ _checkpoint:
 
           pTask->chkInfo.checkpointId = checkpointId;  // 1pTask->checkpointingId;
           pTask->chkInfo.checkpointVer = pItem->submitReqVer;
-          pTask->info.triggerParam = pItem->fetchResultVer;
+          pTask->info.delaySchedParam = pItem->fetchResultVer;
           pTask->info.taskLevel = TASK_LEVEL_SMA;
 
           if (!checkpointBuilt) {
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index d8460d2cb5..1e564ba467 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -770,19 +770,19 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
     tqInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
            " nextProcessVer:%" PRId64
            " child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x "
-           "trigger:%" PRId64 " ms, inputVer:%" PRId64,
+           "delaySched:%" PRId64 " ms, inputVer:%" PRId64,
            vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
            pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
-           (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam, nextProcessVer);
+           (int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam, nextProcessVer);
   } else {
     tqInfo(
         "vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
         " nextProcessVer:%" PRId64
-        " child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64
+        " child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x delaySched:%" PRId64
         " ms, inputVer:%" PRId64,
         vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
         pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
-        (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam, nextProcessVer);
+        (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam, nextProcessVer);
 
     ASSERT(pChkInfo->checkpointVer <= pChkInfo->nextProcessVer);
   }
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 10db53ea38..d6cd5b528d 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -145,9 +145,6 @@ void    destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups);
 int32_t getNumOfDispatchBranch(SStreamTask* pTask);
 void    clearBufferedDispatchMsg(SStreamTask* pTask);
 
-int32_t streamTaskBuildAndSendTriggerMsg(SStreamTask* pTask, const SStreamDataBlock* pData, int32_t dstTaskId,
-                                         int32_t vgId, SEpSet* pEpset);
-
 int32_t           streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
 SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg);
 SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize,
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 522ac9a910..a6cfbb063b 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -227,28 +227,6 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) {
   pMsgInfo->dispatchMsgType = 0;
 }
 
-int32_t streamTaskBuildAndSendTriggerMsg(SStreamTask* pTask, const SStreamDataBlock* pData, int32_t dstTaskId,
-                                         int32_t vgId, SEpSet* pEpset) {
-  SStreamDispatchReq* pReq = taosMemoryCalloc(1, sizeof(SStreamDispatchReq));
-
-  int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
-  int32_t code = tInitStreamDispatchReq(pReq, pTask, pData->srcVgId, numOfBlocks, dstTaskId, pData->type);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
-  }
-
-  for (int32_t i = 0; i < numOfBlocks; i++) {
-    SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i);
-    code = streamAddBlockIntoDispatchMsg(pDataBlock, pReq);
-    if (code != TSDB_CODE_SUCCESS) {
-      destroyDispatchMsg(pReq, 1);
-      return code;
-    }
-  }
-
-  return doSendDispatchMsg(pTask, pReq, vgId, pEpset);
-}
-
 static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) {
   int32_t code = 0;
   int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index e0822f60e7..ae1d86e317 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -458,10 +458,10 @@ void streamMetaClear(SStreamMeta* pMeta) {
     SStreamTask* p = *(SStreamTask**)pIter;
 
     // release the ref by timer
-    if (p->info.triggerParam != 0 && p->info.fillHistory == 0) {  // one more ref in timer
+    if (p->info.delaySchedParam != 0 && p->info.fillHistory == 0) {  // one more ref in timer
       stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", p->id.idStr, p->refCnt);
       taosTmrStop(p->schedInfo.pDelayTimer);
-      p->info.triggerParam = 0;
+      p->info.delaySchedParam = 0;
       streamMetaReleaseTask(pMeta, p);
     }
 
@@ -752,10 +752,10 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
 
     ASSERT(pTask->status.timerActive == 0);
 
-    if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) {
+    if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) {
       stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt);
       taosTmrStop(pTask->schedInfo.pDelayTimer);
-      pTask->info.triggerParam = 0;
+      pTask->info.delaySchedParam = 0;
       streamMetaReleaseTask(pMeta, pTask);
     }
 
diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c
index 461d53d5a9..9c5c230a3d 100644
--- a/source/libs/stream/src/streamQueue.c
+++ b/source/libs/stream/src/streamQueue.c
@@ -330,7 +330,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
   }
 
   if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER &&
-      (pTask->info.triggerParam != 0)) {
+      (pTask->info.delaySchedParam != 0)) {
     atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE);
     stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, pTask->schedInfo.status);
   }
diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c
index 9bd12a4fd8..9c817d565b 100644
--- a/source/libs/stream/src/streamSched.c
+++ b/source/libs/stream/src/streamSched.c
@@ -20,13 +20,13 @@ static void streamTaskResumeHelper(void* param, void* tmrId);
 static void streamTaskSchedHelper(void* param, void* tmrId);
 
 int32_t streamSetupScheduleTrigger(SStreamTask* pTask) {
-  if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) {
+  if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) {
     int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1);
     ASSERT(ref == 2 && pTask->schedInfo.pDelayTimer == NULL);
 
-    stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.triggerParam);
+    stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.delaySchedParam);
 
-    pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.triggerParam, pTask, streamTimer);
+    pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer);
     pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE;
   }
 
@@ -119,7 +119,7 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
 void streamTaskSchedHelper(void* param, void* tmrId) {
   SStreamTask* pTask = (void*)param;
   const char*  id = pTask->id.idStr;
-  int32_t      nextTrigger = (int32_t)pTask->info.triggerParam;
+  int32_t      nextTrigger = (int32_t)pTask->info.delaySchedParam;
 
   int8_t status = atomic_load_8(&pTask->schedInfo.status);
   stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 6d30cc6759..3daadda687 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -107,7 +107,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
 
   pTask->info.taskLevel = taskLevel;
   pTask->info.fillHistory = fillHistory;
-  pTask->info.triggerParam = triggerParam;
+  pTask->info.delaySchedParam = triggerParam;
   pTask->subtableWithoutMd5 = subtableWithoutMd5;
 
   pTask->status.pSM = streamCreateStateMachine(pTask);
diff --git a/source/libs/stream/src/streammsg.c b/source/libs/stream/src/streammsg.c
index f8228a8f5f..705406f044 100644
--- a/source/libs/stream/src/streammsg.c
+++ b/source/libs/stream/src/streammsg.c
@@ -505,7 +505,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
     if (tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo) < 0) return -1;
     if (tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1;
   }
-  if (tEncodeI64(pEncoder, pTask->info.triggerParam) < 0) return -1;
+  if (tEncodeI64(pEncoder, pTask->info.delaySchedParam) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->subtableWithoutMd5) < 0) return -1;
   if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1;
 
@@ -588,7 +588,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
     if (tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo) < 0) return -1;
     if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1;
   }
-  if (tDecodeI64(pDecoder, &pTask->info.triggerParam) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pTask->info.delaySchedParam) < 0) return -1;
   if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER){
     if (tDecodeI8(pDecoder, &pTask->subtableWithoutMd5) < 0) return -1;
   }

From b364379671a805e413350db64755f954fbd8d3c1 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 02:31:29 +0000
Subject: [PATCH 085/132] add compress

---
 source/dnode/vnode/src/meta/metaTable.c | 27 +++++++++----------------
 1 file changed, 9 insertions(+), 18 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index e8ce39f9fb..5bce7937bf 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -36,7 +36,7 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
 
-int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add) {
+int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) {
   int32_t nCols = pWp->nCols;
   int32_t ver = pWp->version;
   if (add) {
@@ -45,7 +45,7 @@ int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add) {
 
     SColCmpr *pCol = p + nCols;
     pCol->id = pSchema->colId;
-    pCol->alg = createDefaultColCmprByType(pSchema->type);
+    pCol->alg = compress;
     pWp->nCols = nCols + 1;
     pWp->version = ver;
     pWp->pColCmpr = p;
@@ -1523,21 +1523,12 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         (void)tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type);
       }
       SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1];
-      if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
-        
-        // if (pAlterTbReq->colCmpr.nCols != pSchema->nCols) {
-        //   terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
-        //   goto _err;
-        // }
-        // updataTableColCmpr(&entry.colCmpr, pCol, 1);
-        // freeColCmpr = true;
-        // ASSERT(entry.colCmpr.nCols == pSchema->nCols);
-      } else {
-        updataTableColCmpr(&entry.colCmpr, pCol, 1);
-        freeColCmpr = true;
-        ASSERT(entry.colCmpr.nCols == pSchema->nCols);
-        break;
-      }
+      uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type)
+                                                                             : pAlterTbReq->compress;
+      updataTableColCmpr(&entry.colCmpr, pCol, 1, compress);
+      freeColCmpr = true;
+      ASSERT(entry.colCmpr.nCols == pSchema->nCols);
+      break;
     case TSDB_ALTER_TABLE_DROP_COLUMN:
       if (pColumn == NULL) {
         terrno = TSDB_CODE_VND_COL_NOT_EXISTS;
@@ -1572,7 +1563,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         (void)tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey);
       }
 
-      updataTableColCmpr(&entry.colCmpr, &tScheam, 0);
+      updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0);
       ASSERT(entry.colCmpr.nCols == pSchema->nCols);
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:

From a1ccf91298a666d80a298689f4ac3c6abca02fb2 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 02:47:57 +0000
Subject: [PATCH 086/132] support add column compress

---
 source/client/src/clientRawBlockWrite.c | 39 +++++++++++++++++++++++++
 source/dnode/mnode/impl/src/mndStb.c    |  2 +-
 source/dnode/vnode/src/meta/metaTable.c |  1 +
 3 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index c5069d46aa..49dfdf6ad2 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -199,6 +199,26 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
       }
       break;
     }
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
+      SFieldWithOptions* field = taosArrayGet(req.pFields, 0);
+      cJSON*             colName = cJSON_CreateString(field->name);
+      cJSON_AddItemToObject(json, "colName", colName);
+      cJSON* colType = cJSON_CreateNumber(field->type);
+      cJSON_AddItemToObject(json, "colType", colType);
+
+      if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY ||
+          field->type == TSDB_DATA_TYPE_GEOMETRY) {
+        int32_t length = field->bytes - VARSTR_HEADER_SIZE;
+        cJSON*  cbytes = cJSON_CreateNumber(length);
+        cJSON_AddItemToObject(json, "colLength", cbytes);
+      } else if (field->type == TSDB_DATA_TYPE_NCHAR) {
+        int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+        cJSON*  cbytes = cJSON_CreateNumber(length);
+        cJSON_AddItemToObject(json, "colLength", cbytes);
+      }
+      setCompressOption(json, field->compress);
+      break;
+    }
     case TSDB_ALTER_TABLE_DROP_TAG:
     case TSDB_ALTER_TABLE_DROP_COLUMN: {
       TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
@@ -546,6 +566,25 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
       }
       break;
     }
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
+      cJSON* colName = cJSON_CreateString(vAlterTbReq.colName);
+      cJSON_AddItemToObject(json, "colName", colName);
+      cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type);
+      cJSON_AddItemToObject(json, "colType", colType);
+
+      if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY ||
+          vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) {
+        int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE;
+        cJSON*  cbytes = cJSON_CreateNumber(length);
+        cJSON_AddItemToObject(json, "colLength", cbytes);
+      } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) {
+        int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+        cJSON*  cbytes = cJSON_CreateNumber(length);
+        cJSON_AddItemToObject(json, "colLength", cbytes);
+      }
+      setCompressOption(json, vAlterTbReq.compress);
+      break;
+    }
     case TSDB_ALTER_TABLE_DROP_COLUMN: {
       cJSON* colName = cJSON_CreateString(vAlterTbReq.colName);
       cJSON_AddItemToObject(json, "colName", colName);
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 1ac9200357..0d3affe6e5 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -1828,7 +1828,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
 
       SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
       pCmpr->id = pSchema->colId;
-      pCmpr->alg = createDefaultColCmprByType(pSchema->type);
+      pCmpr->alg = pField->compress;
       mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
     } else {
       SField *pField = taosArrayGet(pFields, i);
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 5bce7937bf..2a122081b8 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -2272,6 +2272,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta
   pMeta->changed = true;
   switch (pReq->action) {
     case TSDB_ALTER_TABLE_ADD_COLUMN:
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
     case TSDB_ALTER_TABLE_DROP_COLUMN:
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:

From 116409d73fe876555a44acf6c81b15f3b0718f2f Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 02:53:36 +0000
Subject: [PATCH 087/132] merge 3.0

---
 include/common/ttokendef.h   |  750 ++++++------
 source/libs/parser/src/sql.c | 2161 +++++++++++++++++++++++++---------
 2 files changed, 1973 insertions(+), 938 deletions(-)

diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h
index e619edf972..0c15fd3b9d 100644
--- a/include/common/ttokendef.h
+++ b/include/common/ttokendef.h
@@ -16,381 +16,381 @@
 #ifndef _TD_COMMON_TOKEN_H_
 #define _TD_COMMON_TOKEN_H_
 
-#define TK_OR                               1
-#define TK_AND                              2
-#define TK_UNION                            3
-#define TK_ALL                              4
-#define TK_MINUS                            5
-#define TK_EXCEPT                           6
-#define TK_INTERSECT                        7
-#define TK_NK_BITAND                        8
-#define TK_NK_BITOR                         9
-#define TK_NK_LSHIFT                       10
-#define TK_NK_RSHIFT                       11
-#define TK_NK_PLUS                         12
-#define TK_NK_MINUS                        13
-#define TK_NK_STAR                         14
-#define TK_NK_SLASH                        15
-#define TK_NK_REM                          16
-#define TK_NK_CONCAT                       17
-#define TK_CREATE                          18
-#define TK_ACCOUNT                         19
-#define TK_NK_ID                           20
-#define TK_PASS                            21
-#define TK_NK_STRING                       22
-#define TK_ALTER                           23
-#define TK_PPS                             24
-#define TK_TSERIES                         25
-#define TK_STORAGE                         26
-#define TK_STREAMS                         27
-#define TK_QTIME                           28
-#define TK_DBS                             29
-#define TK_USERS                           30
-#define TK_CONNS                           31
-#define TK_STATE                           32
-#define TK_NK_COMMA                        33
-#define TK_HOST                            34
-#define TK_USER                            35
-#define TK_ENABLE                          36
-#define TK_NK_INTEGER                      37
-#define TK_SYSINFO                         38
-#define TK_ADD                             39
-#define TK_DROP                            40
-#define TK_GRANT                           41
-#define TK_ON                              42
-#define TK_TO                              43
-#define TK_REVOKE                          44
-#define TK_FROM                            45
-#define TK_SUBSCRIBE                       46
-#define TK_READ                            47
-#define TK_WRITE                           48
-#define TK_NK_DOT                          49
-#define TK_WITH                            50
-#define TK_ENCRYPT_KEY                     51
-#define TK_DNODE                           52
-#define TK_PORT                            53
-#define TK_DNODES                          54
-#define TK_RESTORE                         55
-#define TK_NK_IPTOKEN                      56
-#define TK_FORCE                           57
-#define TK_UNSAFE                          58
-#define TK_CLUSTER                         59
-#define TK_LOCAL                           60
-#define TK_QNODE                           61
-#define TK_BNODE                           62
-#define TK_SNODE                           63
-#define TK_MNODE                           64
-#define TK_VNODE                           65
-#define TK_DATABASE                        66
-#define TK_USE                             67
-#define TK_FLUSH                           68
-#define TK_TRIM                            69
-#define TK_S3MIGRATE                       70
-#define TK_COMPACT                         71
-#define TK_IF                              72
-#define TK_NOT                             73
-#define TK_EXISTS                          74
-#define TK_BUFFER                          75
-#define TK_CACHEMODEL                      76
-#define TK_CACHESIZE                       77
-#define TK_COMP                            78
-#define TK_DURATION                        79
-#define TK_NK_VARIABLE                     80
-#define TK_MAXROWS                         81
-#define TK_MINROWS                         82
-#define TK_KEEP                            83
-#define TK_PAGES                           84
-#define TK_PAGESIZE                        85
-#define TK_TSDB_PAGESIZE                   86
-#define TK_PRECISION                       87
-#define TK_REPLICA                         88
-#define TK_VGROUPS                         89
-#define TK_SINGLE_STABLE                   90
-#define TK_RETENTIONS                      91
-#define TK_SCHEMALESS                      92
-#define TK_WAL_LEVEL                       93
-#define TK_WAL_FSYNC_PERIOD                94
-#define TK_WAL_RETENTION_PERIOD            95
-#define TK_WAL_RETENTION_SIZE              96
-#define TK_WAL_ROLL_PERIOD                 97
-#define TK_WAL_SEGMENT_SIZE                98
-#define TK_STT_TRIGGER                     99
-#define TK_TABLE_PREFIX                   100
-#define TK_TABLE_SUFFIX                   101
-#define TK_S3_CHUNKSIZE                   102
-#define TK_S3_KEEPLOCAL                   103
-#define TK_S3_COMPACT                     104
-#define TK_KEEP_TIME_OFFSET               105
-#define TK_ENCRYPT_ALGORITHM              106
-#define TK_NK_COLON                       107
-#define TK_BWLIMIT                        108
-#define TK_START                          109
-#define TK_TIMESTAMP                      110
-#define TK_END                            111
-#define TK_TABLE                          112
-#define TK_NK_LP                          113
-#define TK_NK_RP                          114
-#define TK_STABLE                         115
-#define TK_COLUMN                         116
-#define TK_MODIFY                         117
-#define TK_RENAME                         118
-#define TK_TAG                            119
-#define TK_SET                            120
-#define TK_NK_EQ                          121
-#define TK_USING                          122
-#define TK_TAGS                           123
-#define TK_BOOL                           124
-#define TK_TINYINT                        125
-#define TK_SMALLINT                       126
-#define TK_INT                            127
-#define TK_INTEGER                        128
-#define TK_BIGINT                         129
-#define TK_FLOAT                          130
-#define TK_DOUBLE                         131
-#define TK_BINARY                         132
-#define TK_NCHAR                          133
-#define TK_UNSIGNED                       134
-#define TK_JSON                           135
-#define TK_VARCHAR                        136
-#define TK_MEDIUMBLOB                     137
-#define TK_BLOB                           138
-#define TK_VARBINARY                      139
-#define TK_GEOMETRY                       140
-#define TK_DECIMAL                        141
-#define TK_COMMENT                        142
-#define TK_MAX_DELAY                      143
-#define TK_WATERMARK                      144
-#define TK_ROLLUP                         145
-#define TK_TTL                            146
-#define TK_SMA                            147
-#define TK_DELETE_MARK                    148
-#define TK_FIRST                          149
-#define TK_LAST                           150
-#define TK_SHOW                           151
-#define TK_PRIVILEGES                     152
-#define TK_DATABASES                      153
-#define TK_TABLES                         154
-#define TK_STABLES                        155
-#define TK_MNODES                         156
-#define TK_QNODES                         157
-#define TK_ARBGROUPS                      158
-#define TK_FUNCTIONS                      159
-#define TK_INDEXES                        160
-#define TK_ACCOUNTS                       161
-#define TK_APPS                           162
-#define TK_CONNECTIONS                    163
-#define TK_LICENCES                       164
-#define TK_GRANTS                         165
-#define TK_FULL                           166
-#define TK_LOGS                           167
-#define TK_MACHINES                       168
-#define TK_ENCRYPTIONS                    169
-#define TK_QUERIES                        170
-#define TK_SCORES                         171
-#define TK_TOPICS                         172
-#define TK_VARIABLES                      173
-#define TK_BNODES                         174
-#define TK_SNODES                         175
-#define TK_TRANSACTIONS                   176
-#define TK_DISTRIBUTED                    177
-#define TK_CONSUMERS                      178
-#define TK_SUBSCRIPTIONS                  179
-#define TK_VNODES                         180
-#define TK_ALIVE                          181
-#define TK_VIEWS                          182
-#define TK_VIEW                           183
-#define TK_COMPACTS                       184
-#define TK_NORMAL                         185
-#define TK_CHILD                          186
-#define TK_LIKE                           187
-#define TK_TBNAME                         188
-#define TK_QTAGS                          189
-#define TK_AS                             190
-#define TK_SYSTEM                         191
-#define TK_TSMA                           192
-#define TK_INTERVAL                       193
-#define TK_RECURSIVE                      194
-#define TK_TSMAS                          195
-#define TK_FUNCTION                       196
-#define TK_INDEX                          197
-#define TK_COUNT                          198
-#define TK_LAST_ROW                       199
-#define TK_META                           200
-#define TK_ONLY                           201
-#define TK_TOPIC                          202
-#define TK_CONSUMER                       203
-#define TK_GROUP                          204
-#define TK_DESC                           205
-#define TK_DESCRIBE                       206
-#define TK_RESET                          207
-#define TK_QUERY                          208
-#define TK_CACHE                          209
-#define TK_EXPLAIN                        210
-#define TK_ANALYZE                        211
-#define TK_VERBOSE                        212
-#define TK_NK_BOOL                        213
-#define TK_RATIO                          214
-#define TK_NK_FLOAT                       215
-#define TK_OUTPUTTYPE                     216
-#define TK_AGGREGATE                      217
-#define TK_BUFSIZE                        218
-#define TK_LANGUAGE                       219
-#define TK_REPLACE                        220
-#define TK_STREAM                         221
-#define TK_INTO                           222
-#define TK_PAUSE                          223
-#define TK_RESUME                         224
-#define TK_PRIMARY                        225
-#define TK_KEY                            226
-#define TK_TRIGGER                        227
-#define TK_AT_ONCE                        228
-#define TK_WINDOW_CLOSE                   229
-#define TK_IGNORE                         230
-#define TK_EXPIRED                        231
-#define TK_FILL_HISTORY                   232
-#define TK_UPDATE                         233
-#define TK_SUBTABLE                       234
-#define TK_UNTREATED                      235
-#define TK_KILL                           236
-#define TK_CONNECTION                     237
-#define TK_TRANSACTION                    238
-#define TK_BALANCE                        239
-#define TK_VGROUP                         240
-#define TK_LEADER                         241
-#define TK_MERGE                          242
-#define TK_REDISTRIBUTE                   243
-#define TK_SPLIT                          244
-#define TK_DELETE                         245
-#define TK_INSERT                         246
-#define TK_NK_BIN                         247
-#define TK_NK_HEX                         248
-#define TK_NULL                           249
-#define TK_NK_QUESTION                    250
-#define TK_NK_ALIAS                       251
-#define TK_NK_ARROW                       252
-#define TK_ROWTS                          253
-#define TK_QSTART                         254
-#define TK_QEND                           255
-#define TK_QDURATION                      256
-#define TK_WSTART                         257
-#define TK_WEND                           258
-#define TK_WDURATION                      259
-#define TK_IROWTS                         260
-#define TK_ISFILLED                       261
-#define TK_CAST                           262
-#define TK_NOW                            263
-#define TK_TODAY                          264
-#define TK_TIMEZONE                       265
-#define TK_CLIENT_VERSION                 266
-#define TK_SERVER_VERSION                 267
-#define TK_SERVER_STATUS                  268
-#define TK_CURRENT_USER                   269
-#define TK_CASE                           270
-#define TK_WHEN                           271
-#define TK_THEN                           272
-#define TK_ELSE                           273
-#define TK_BETWEEN                        274
-#define TK_IS                             275
-#define TK_NK_LT                          276
-#define TK_NK_GT                          277
-#define TK_NK_LE                          278
-#define TK_NK_GE                          279
-#define TK_NK_NE                          280
-#define TK_MATCH                          281
-#define TK_NMATCH                         282
-#define TK_CONTAINS                       283
-#define TK_IN                             284
-#define TK_JOIN                           285
-#define TK_INNER                          286
-#define TK_LEFT                           287
-#define TK_RIGHT                          288
-#define TK_OUTER                          289
-#define TK_SEMI                           290
-#define TK_ANTI                           291
-#define TK_ASOF                           292
-#define TK_WINDOW                         293
-#define TK_WINDOW_OFFSET                  294
-#define TK_JLIMIT                         295
-#define TK_SELECT                         296
-#define TK_NK_HINT                        297
-#define TK_DISTINCT                       298
-#define TK_WHERE                          299
-#define TK_PARTITION                      300
-#define TK_BY                             301
-#define TK_SESSION                        302
-#define TK_STATE_WINDOW                   303
-#define TK_EVENT_WINDOW                   304
-#define TK_COUNT_WINDOW                   305
-#define TK_SLIDING                        306
-#define TK_FILL                           307
-#define TK_VALUE                          308
-#define TK_VALUE_F                        309
-#define TK_NONE                           310
-#define TK_PREV                           311
-#define TK_NULL_F                         312
-#define TK_LINEAR                         313
-#define TK_NEXT                           314
-#define TK_HAVING                         315
-#define TK_RANGE                          316
-#define TK_EVERY                          317
-#define TK_ORDER                          318
-#define TK_SLIMIT                         319
-#define TK_SOFFSET                        320
-#define TK_LIMIT                          321
-#define TK_OFFSET                         322
-#define TK_ASC                            323
-#define TK_NULLS                          324
-#define TK_ABORT                          325
-#define TK_AFTER                          326
-#define TK_ATTACH                         327
-#define TK_BEFORE                         328
-#define TK_BEGIN                          329
-#define TK_BITAND                         330
-#define TK_BITNOT                         331
-#define TK_BITOR                          332
-#define TK_BLOCKS                         333
-#define TK_CHANGE                         334
-#define TK_COMMA                          335
-#define TK_CONCAT                         336
-#define TK_CONFLICT                       337
-#define TK_COPY                           338
-#define TK_DEFERRED                       339
-#define TK_DELIMITERS                     340
-#define TK_DETACH                         341
-#define TK_DIVIDE                         342
-#define TK_DOT                            343
-#define TK_EACH                           344
-#define TK_FAIL                           345
-#define TK_FILE                           346
-#define TK_FOR                            347
-#define TK_GLOB                           348
-#define TK_ID                             349
-#define TK_IMMEDIATE                      350
-#define TK_IMPORT                         351
-#define TK_INITIALLY                      352
-#define TK_INSTEAD                        353
-#define TK_ISNULL                         354
-#define TK_MODULES                        355
-#define TK_NK_BITNOT                      356
-#define TK_NK_SEMI                        357
-#define TK_NOTNULL                        358
-#define TK_OF                             359
-#define TK_PLUS                           360
-#define TK_PRIVILEGE                      361
-#define TK_RAISE                          362
-#define TK_RESTRICT                       363
-#define TK_ROW                            364
-#define TK_STAR                           365
-#define TK_STATEMENT                      366
-#define TK_STRICT                         367
-#define TK_STRING                         368
-#define TK_TIMES                          369
-#define TK_VALUES                         370
-#define TK_VARIABLE                       371
-#define TK_WAL                            372
-#define TK_ENCODE                         373
-#define TK_COMPRESS                       374
-#define TK_LEVEL                          375
+#define TK_OR                   1
+#define TK_AND                  2
+#define TK_UNION                3
+#define TK_ALL                  4
+#define TK_MINUS                5
+#define TK_EXCEPT               6
+#define TK_INTERSECT            7
+#define TK_NK_BITAND            8
+#define TK_NK_BITOR             9
+#define TK_NK_LSHIFT            10
+#define TK_NK_RSHIFT            11
+#define TK_NK_PLUS              12
+#define TK_NK_MINUS             13
+#define TK_NK_STAR              14
+#define TK_NK_SLASH             15
+#define TK_NK_REM               16
+#define TK_NK_CONCAT            17
+#define TK_CREATE               18
+#define TK_ACCOUNT              19
+#define TK_NK_ID                20
+#define TK_PASS                 21
+#define TK_NK_STRING            22
+#define TK_ALTER                23
+#define TK_PPS                  24
+#define TK_TSERIES              25
+#define TK_STORAGE              26
+#define TK_STREAMS              27
+#define TK_QTIME                28
+#define TK_DBS                  29
+#define TK_USERS                30
+#define TK_CONNS                31
+#define TK_STATE                32
+#define TK_NK_COMMA             33
+#define TK_HOST                 34
+#define TK_USER                 35
+#define TK_ENABLE               36
+#define TK_NK_INTEGER           37
+#define TK_SYSINFO              38
+#define TK_ADD                  39
+#define TK_DROP                 40
+#define TK_GRANT                41
+#define TK_ON                   42
+#define TK_TO                   43
+#define TK_REVOKE               44
+#define TK_FROM                 45
+#define TK_SUBSCRIBE            46
+#define TK_READ                 47
+#define TK_WRITE                48
+#define TK_NK_DOT               49
+#define TK_WITH                 50
+#define TK_ENCRYPT_KEY          51
+#define TK_DNODE                52
+#define TK_PORT                 53
+#define TK_DNODES               54
+#define TK_RESTORE              55
+#define TK_NK_IPTOKEN           56
+#define TK_FORCE                57
+#define TK_UNSAFE               58
+#define TK_CLUSTER              59
+#define TK_LOCAL                60
+#define TK_QNODE                61
+#define TK_BNODE                62
+#define TK_SNODE                63
+#define TK_MNODE                64
+#define TK_VNODE                65
+#define TK_DATABASE             66
+#define TK_USE                  67
+#define TK_FLUSH                68
+#define TK_TRIM                 69
+#define TK_S3MIGRATE            70
+#define TK_COMPACT              71
+#define TK_IF                   72
+#define TK_NOT                  73
+#define TK_EXISTS               74
+#define TK_BUFFER               75
+#define TK_CACHEMODEL           76
+#define TK_CACHESIZE            77
+#define TK_COMP                 78
+#define TK_DURATION             79
+#define TK_NK_VARIABLE          80
+#define TK_MAXROWS              81
+#define TK_MINROWS              82
+#define TK_KEEP                 83
+#define TK_PAGES                84
+#define TK_PAGESIZE             85
+#define TK_TSDB_PAGESIZE        86
+#define TK_PRECISION            87
+#define TK_REPLICA              88
+#define TK_VGROUPS              89
+#define TK_SINGLE_STABLE        90
+#define TK_RETENTIONS           91
+#define TK_SCHEMALESS           92
+#define TK_WAL_LEVEL            93
+#define TK_WAL_FSYNC_PERIOD     94
+#define TK_WAL_RETENTION_PERIOD 95
+#define TK_WAL_RETENTION_SIZE   96
+#define TK_WAL_ROLL_PERIOD      97
+#define TK_WAL_SEGMENT_SIZE     98
+#define TK_STT_TRIGGER          99
+#define TK_TABLE_PREFIX         100
+#define TK_TABLE_SUFFIX         101
+#define TK_S3_CHUNKSIZE         102
+#define TK_S3_KEEPLOCAL         103
+#define TK_S3_COMPACT           104
+#define TK_KEEP_TIME_OFFSET     105
+#define TK_ENCRYPT_ALGORITHM    106
+#define TK_NK_COLON             107
+#define TK_BWLIMIT              108
+#define TK_START                109
+#define TK_TIMESTAMP            110
+#define TK_END                  111
+#define TK_TABLE                112
+#define TK_NK_LP                113
+#define TK_NK_RP                114
+#define TK_STABLE               115
+#define TK_COLUMN               116
+#define TK_MODIFY               117
+#define TK_RENAME               118
+#define TK_TAG                  119
+#define TK_SET                  120
+#define TK_NK_EQ                121
+#define TK_USING                122
+#define TK_TAGS                 123
+#define TK_BOOL                 124
+#define TK_TINYINT              125
+#define TK_SMALLINT             126
+#define TK_INT                  127
+#define TK_INTEGER              128
+#define TK_BIGINT               129
+#define TK_FLOAT                130
+#define TK_DOUBLE               131
+#define TK_BINARY               132
+#define TK_NCHAR                133
+#define TK_UNSIGNED             134
+#define TK_JSON                 135
+#define TK_VARCHAR              136
+#define TK_MEDIUMBLOB           137
+#define TK_BLOB                 138
+#define TK_VARBINARY            139
+#define TK_GEOMETRY             140
+#define TK_DECIMAL              141
+#define TK_COMMENT              142
+#define TK_MAX_DELAY            143
+#define TK_WATERMARK            144
+#define TK_ROLLUP               145
+#define TK_TTL                  146
+#define TK_SMA                  147
+#define TK_DELETE_MARK          148
+#define TK_FIRST                149
+#define TK_LAST                 150
+#define TK_SHOW                 151
+#define TK_PRIVILEGES           152
+#define TK_DATABASES            153
+#define TK_TABLES               154
+#define TK_STABLES              155
+#define TK_MNODES               156
+#define TK_QNODES               157
+#define TK_ARBGROUPS            158
+#define TK_FUNCTIONS            159
+#define TK_INDEXES              160
+#define TK_ACCOUNTS             161
+#define TK_APPS                 162
+#define TK_CONNECTIONS          163
+#define TK_LICENCES             164
+#define TK_GRANTS               165
+#define TK_FULL                 166
+#define TK_LOGS                 167
+#define TK_MACHINES             168
+#define TK_ENCRYPTIONS          169
+#define TK_QUERIES              170
+#define TK_SCORES               171
+#define TK_TOPICS               172
+#define TK_VARIABLES            173
+#define TK_BNODES               174
+#define TK_SNODES               175
+#define TK_TRANSACTIONS         176
+#define TK_DISTRIBUTED          177
+#define TK_CONSUMERS            178
+#define TK_SUBSCRIPTIONS        179
+#define TK_VNODES               180
+#define TK_ALIVE                181
+#define TK_VIEWS                182
+#define TK_VIEW                 183
+#define TK_COMPACTS             184
+#define TK_NORMAL               185
+#define TK_CHILD                186
+#define TK_LIKE                 187
+#define TK_TBNAME               188
+#define TK_QTAGS                189
+#define TK_AS                   190
+#define TK_SYSTEM               191
+#define TK_TSMA                 192
+#define TK_INTERVAL             193
+#define TK_RECURSIVE            194
+#define TK_TSMAS                195
+#define TK_FUNCTION             196
+#define TK_INDEX                197
+#define TK_COUNT                198
+#define TK_LAST_ROW             199
+#define TK_META                 200
+#define TK_ONLY                 201
+#define TK_TOPIC                202
+#define TK_CONSUMER             203
+#define TK_GROUP                204
+#define TK_DESC                 205
+#define TK_DESCRIBE             206
+#define TK_RESET                207
+#define TK_QUERY                208
+#define TK_CACHE                209
+#define TK_EXPLAIN              210
+#define TK_ANALYZE              211
+#define TK_VERBOSE              212
+#define TK_NK_BOOL              213
+#define TK_RATIO                214
+#define TK_NK_FLOAT             215
+#define TK_OUTPUTTYPE           216
+#define TK_AGGREGATE            217
+#define TK_BUFSIZE              218
+#define TK_LANGUAGE             219
+#define TK_REPLACE              220
+#define TK_STREAM               221
+#define TK_INTO                 222
+#define TK_PAUSE                223
+#define TK_RESUME               224
+#define TK_PRIMARY              225
+#define TK_KEY                  226
+#define TK_TRIGGER              227
+#define TK_AT_ONCE              228
+#define TK_WINDOW_CLOSE         229
+#define TK_IGNORE               230
+#define TK_EXPIRED              231
+#define TK_FILL_HISTORY         232
+#define TK_UPDATE               233
+#define TK_SUBTABLE             234
+#define TK_UNTREATED            235
+#define TK_KILL                 236
+#define TK_CONNECTION           237
+#define TK_TRANSACTION          238
+#define TK_BALANCE              239
+#define TK_VGROUP               240
+#define TK_LEADER               241
+#define TK_MERGE                242
+#define TK_REDISTRIBUTE         243
+#define TK_SPLIT                244
+#define TK_DELETE               245
+#define TK_INSERT               246
+#define TK_NK_BIN               247
+#define TK_NK_HEX               248
+#define TK_NULL                 249
+#define TK_NK_QUESTION          250
+#define TK_NK_ALIAS             251
+#define TK_NK_ARROW             252
+#define TK_ROWTS                253
+#define TK_QSTART               254
+#define TK_QEND                 255
+#define TK_QDURATION            256
+#define TK_WSTART               257
+#define TK_WEND                 258
+#define TK_WDURATION            259
+#define TK_IROWTS               260
+#define TK_ISFILLED             261
+#define TK_CAST                 262
+#define TK_NOW                  263
+#define TK_TODAY                264
+#define TK_TIMEZONE             265
+#define TK_CLIENT_VERSION       266
+#define TK_SERVER_VERSION       267
+#define TK_SERVER_STATUS        268
+#define TK_CURRENT_USER         269
+#define TK_CASE                 270
+#define TK_WHEN                 271
+#define TK_THEN                 272
+#define TK_ELSE                 273
+#define TK_BETWEEN              274
+#define TK_IS                   275
+#define TK_NK_LT                276
+#define TK_NK_GT                277
+#define TK_NK_LE                278
+#define TK_NK_GE                279
+#define TK_NK_NE                280
+#define TK_MATCH                281
+#define TK_NMATCH               282
+#define TK_CONTAINS             283
+#define TK_IN                   284
+#define TK_JOIN                 285
+#define TK_INNER                286
+#define TK_LEFT                 287
+#define TK_RIGHT                288
+#define TK_OUTER                289
+#define TK_SEMI                 290
+#define TK_ANTI                 291
+#define TK_ASOF                 292
+#define TK_WINDOW               293
+#define TK_WINDOW_OFFSET        294
+#define TK_JLIMIT               295
+#define TK_SELECT               296
+#define TK_NK_HINT              297
+#define TK_DISTINCT             298
+#define TK_WHERE                299
+#define TK_PARTITION            300
+#define TK_BY                   301
+#define TK_SESSION              302
+#define TK_STATE_WINDOW         303
+#define TK_EVENT_WINDOW         304
+#define TK_COUNT_WINDOW         305
+#define TK_SLIDING              306
+#define TK_FILL                 307
+#define TK_VALUE                308
+#define TK_VALUE_F              309
+#define TK_NONE                 310
+#define TK_PREV                 311
+#define TK_NULL_F               312
+#define TK_LINEAR               313
+#define TK_NEXT                 314
+#define TK_HAVING               315
+#define TK_RANGE                316
+#define TK_EVERY                317
+#define TK_ORDER                318
+#define TK_SLIMIT               319
+#define TK_SOFFSET              320
+#define TK_LIMIT                321
+#define TK_OFFSET               322
+#define TK_ASC                  323
+#define TK_NULLS                324
+#define TK_ABORT                325
+#define TK_AFTER                326
+#define TK_ATTACH               327
+#define TK_BEFORE               328
+#define TK_BEGIN                329
+#define TK_BITAND               330
+#define TK_BITNOT               331
+#define TK_BITOR                332
+#define TK_BLOCKS               333
+#define TK_CHANGE               334
+#define TK_COMMA                335
+#define TK_CONCAT               336
+#define TK_CONFLICT             337
+#define TK_COPY                 338
+#define TK_DEFERRED             339
+#define TK_DELIMITERS           340
+#define TK_DETACH               341
+#define TK_DIVIDE               342
+#define TK_DOT                  343
+#define TK_EACH                 344
+#define TK_FAIL                 345
+#define TK_FILE                 346
+#define TK_FOR                  347
+#define TK_GLOB                 348
+#define TK_ID                   349
+#define TK_IMMEDIATE            350
+#define TK_IMPORT               351
+#define TK_INITIALLY            352
+#define TK_INSTEAD              353
+#define TK_ISNULL               354
+#define TK_MODULES              355
+#define TK_NK_BITNOT            356
+#define TK_NK_SEMI              357
+#define TK_NOTNULL              358
+#define TK_OF                   359
+#define TK_PLUS                 360
+#define TK_PRIVILEGE            361
+#define TK_RAISE                362
+#define TK_RESTRICT             363
+#define TK_ROW                  364
+#define TK_STAR                 365
+#define TK_STATEMENT            366
+#define TK_STRICT               367
+#define TK_STRING               368
+#define TK_TIMES                369
+#define TK_VALUES               370
+#define TK_VARIABLE             371
+#define TK_WAL                  372
+#define TK_ENCODE               373
+#define TK_COMPRESS             374
+#define TK_LEVEL                375
 
 #define TK_NK_SPACE   600
 #define TK_NK_COMMENT 601
diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c
index 52dd135cd3..226696532b 100644
--- a/source/libs/parser/src/sql.c
+++ b/source/libs/parser/src/sql.c
@@ -25,6 +25,8 @@
 ** input grammar file:
 */
 /************ Begin %include sections from the grammar ************************/
+#line 11 "sql.y"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -40,6 +42,7 @@
 #include "parAst.h"
 
 #define YYSTACKDEPTH 0
+#line 46 "sql.c"
 /**************** End of %include directives **********************************/
 /* These constants specify the various numeric values for terminal symbols.
 ***************** Begin token definitions *************************************/
@@ -517,18 +520,18 @@ typedef union {
 #define ParseCTX_FETCH
 #define ParseCTX_STORE
 #define YYFALLBACK 1
-#define YYNSTATE             963
+#define YYNSTATE             965
 #define YYNRULE              748
 #define YYNRULE_WITH_ACTION  748
 #define YYNTOKEN             376
-#define YY_MAX_SHIFT         962
-#define YY_MIN_SHIFTREDUCE   1431
-#define YY_MAX_SHIFTREDUCE   2178
-#define YY_ERROR_ACTION      2179
-#define YY_ACCEPT_ACTION     2180
-#define YY_NO_ACTION         2181
-#define YY_MIN_REDUCE        2182
-#define YY_MAX_REDUCE        2929
+#define YY_MAX_SHIFT         964
+#define YY_MIN_SHIFTREDUCE   1432
+#define YY_MAX_SHIFTREDUCE   2179
+#define YY_ERROR_ACTION      2180
+#define YY_ACCEPT_ACTION     2181
+#define YY_NO_ACTION         2182
+#define YY_MIN_REDUCE        2183
+#define YY_MAX_REDUCE        2930
 /************* End control #defines *******************************************/
 #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])))
 
@@ -597,329 +600,329 @@ typedef union {
 *********** Begin parsing tables **********************************************/
 #define YY_ACTTAB_COUNT (3228)
 static const YYACTIONTYPE yy_action[] = {
- /*     0 */    37,  338, 2538, 2705,  639,  542, 2373,  640, 2225,  190,
- /*    10 */  2680,  315,   47,   45, 2095, 2812,  803,  464,  494,  493,
- /*    20 */   469, 2183, 1916, 2535,  845,   40,   39, 2705,   66,   46,
- /*    30 */    44,   43,   42,   41, 1941, 2004, 1941, 1914, 2684, 2480,
- /*    40 */   838, 2809,  136, 1923, 2723,  135,  134,  133,  132,  131,
- /*    50 */   130,  129,  128,  127,  604,  602,  831,  409,  854,  372,
- /*    60 */  2670,  230,  840, 2368,  417,   33, 1999,  830, 2723,  800,
+ /*     0 */    37,  338, 2539, 2706,  641,  544, 2374,  642, 2226,  190,
+ /*    10 */  2681,  315,   47,   45, 2096, 2813,  805,  466,  496,  495,
+ /*    20 */   471, 2184, 1917, 2536,  847,   40,   39, 2706,   66,   46,
+ /*    30 */    44,   43,   42,   41, 1942, 2005, 1942, 1915, 2685, 2481,
+ /*    40 */   840, 2810,  136, 1924, 2724,  135,  134,  133,  132,  131,
+ /*    50 */   130,  129,  128,  127,  606,  604,  833,  410,  856,  372,
+ /*    60 */  2671,  230,  842, 2369,  418,   33, 2000,  832, 2724,  802,
  /*    70 */   155,   40,   39,   19,  301,   46,   44,   43,   42,   41,
- /*    80 */  1922, 2269,  800,  155, 2670,  210,  840,  724,  136, 2686,
- /*    90 */  2689,  135,  134,  133,  132,  131,  130,  129,  128,  127,
- /*   100 */   859, 2431,  722,  252,  720,  286,  285,  642, 2704, 2233,
- /*   110 */   959, 2743, 2812,   15, 1942,  119, 2706,  844, 2708, 2709,
- /*   120 */   839,  647,  859,  103,  640, 2225,  197,  199,  854, 2797,
- /*   130 */   644, 2139, 2704,  465, 2793, 2743,  641,   62, 2808,  405,
- /*   140 */  2706,  844, 2708, 2709,  839,  837,  859,  823, 2762, 2006,
- /*   150 */  2007,   50,  437,  218, 2511,  753,  932,  931,  930,  929,
- /*   160 */   497, 2844,  928,  927,  160,  922,  921,  920,  919,  918,
- /*   170 */   917,  916,  159,  910,  909,  908,  496,  495,  905,  904,
- /*   180 */   903,  196,  195,  902,  492,  901,  900,  899, 1977, 1987,
- /*   190 */   802,  185, 2805, 2806,  778,  153, 2810,  290, 2005, 2008,
- /*   200 */   800,  155, 1762, 1763,  192, 2805,  799,  579,  147,  798,
- /*   210 */   788, 2663,  578, 1917,  485, 1915, 2895,    9, 2895, 2099,
- /*   220 */   577, 1926,   40,   39,  737, 1941,   46,   44,   43,   42,
- /*   230 */    41, 2360, 2705, 2205,  787,  217,  787,  217,  122, 2896,
- /*   240 */   789, 2896,  789, 1689, 1690,  841, 2388,  487, 2386, 1920,
- /*   250 */  1921, 1974,  777, 1976, 1979, 1980, 1981, 1982, 1983, 1984,
- /*   260 */  1985, 1986,  836,  857,  856, 1998, 2000, 2001, 2002, 2003,
- /*   270 */     2,   47,   45, 2723,  915,  474,  414, 2344, 1939,  469,
- /*   280 */  2723, 1916, 1843, 1844, 2900,  586,  859,  429,  607, 2670,
- /*   290 */  2670,  840, 2895,  606, 2004,   60, 1914,   40,   39,  453,
- /*   300 */  2585,   46,   44,   43,   42,   41,  751, 2014,  734, 1945,
- /*   310 */   565, 2899,  608, 1941,  197, 2896, 2898,  415,  567, 1498,
- /*   320 */   322, 1497,  125, 2805, 2806, 1999,  153, 2810,  180,  545,
- /*   330 */   855, 2384,   19,  662,  322, 2033,  151, 2704, 2387, 1922,
- /*   340 */  2743, 2070, 2512,  773,  119, 2706,  844, 2708, 2709,  839,
- /*   350 */   221,  859, 1471,  776,  157, 1499,  166, 2768, 2797, 2182,
- /*   360 */    40,   39,  465, 2793,   46,   44,   43,   42,   41,  959,
- /*   370 */   438, 1478,   15,  896,  172,  171,  893,  892,  891,  169,
- /*   380 */  1945,  553, 1941,  145,  144,  143,  142,  141,  140,  139,
- /*   390 */   138,  137,  655, 1978, 1473, 1476, 1477,  535,   62,   40,
- /*   400 */    39, 2034,  534,   46,   44,   43,   42,   41, 2006, 2007,
- /*   410 */   659, 2518, 2497,  658,  594,  593,  592,  591,  590,  585,
- /*   420 */   584,  583,  582,  421,   29, 1974, 2070,  572,  571,  570,
- /*   430 */   569,  568,  562,  561,  560,  254,  555,  554,  436,  642,
- /*   440 */   854, 2233,  546, 1750, 1751,  329,  330, 1977, 1987, 1769,
- /*   450 */   328,  754,  779,  774,  767,  763, 1975, 2005, 2008, 2895,
- /*   460 */  2817, 2067, 2068, 2069, 2817, 2817, 2817, 2817, 2817, 2525,
- /*   470 */  2375,  660, 1917,  519, 1915,   62,  898, 2901,  217,  213,
- /*   480 */   738, 1978, 2896,  789,  855, 2384,  855, 2384,   36,  467,
- /*   490 */  2028, 2029, 2030, 2031, 2032, 2036, 2037, 2038, 2039, 2072,
- /*   500 */  2073, 2074, 2075, 2076,  146, 2140,   55,  376, 1920, 1921,
- /*   510 */  1974,  685, 1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985,
- /*   520 */  1986,  836,  857,  856, 1998, 2000, 2001, 2002, 2003,    2,
- /*   530 */    12,   47,   45,   50,  474, 1925, 2705, 2538,  181,  469,
- /*   540 */  1946, 1916,  116,  389, 1975,  859, 2067, 2068, 2069,  803,
- /*   550 */   208,  113,  471,  598, 2004, 1589, 1914, 2170, 2535,  845,
- /*   560 */   387,   76,  649, 2577,   75, 2705,   40,   39,  452, 2585,
- /*   570 */    46,   44,   43,   42,   41,  416,  735, 2723,  841, 2070,
- /*   580 */  2235,  322, 1942, 1922, 2680, 1999, 2900,  250,  621,  619,
- /*   590 */   616,  614,   19, 2670, 2895,  840,   40,   39, 1591, 1922,
- /*   600 */    46,   44,   43,   42,   41, 2490, 2723,  706,  705,  704,
- /*   610 */   523, 1946, 2684, 2899,  696,  152,  700, 2896, 2897,  241,
- /*   620 */   699, 2538, 2670,  656,  840,  698,  703,  446,  445,  959,
- /*   630 */   912,  697,   15,   62,   98,  444,  693,  692,  691,  525,
- /*   640 */   521, 2704, 2536,  845, 2743,  597,  240,  549,  119, 2706,
- /*   650 */   844, 2708, 2709,  839,  795,  859,  855, 2384,  322,  595,
- /*   660 */   199, 2380, 2797, 2686, 2688,  466,  465, 2793, 2006, 2007,
- /*   670 */  2704,   63, 2111, 2743,  859, 2169,  146,  119, 2706,  844,
- /*   680 */  2708, 2709,  839,  690,  859,   12, 2128,  657, 2531, 2915,
- /*   690 */   183, 2797, 2194,   51, 2845,  465, 2793,  914, 2817, 2067,
- /*   700 */  2068, 2069, 2817, 2817, 2817, 2817, 2817, 1977, 1987,   35,
- /*   710 */  1944, 1789, 1790, 1928, 2180,   40,   39, 2005, 2008,   46,
- /*   720 */    44,   43,   42,   41, 2358,   86,   85,  538, 2438, 2439,
- /*   730 */   229, 2092, 1917, 2444, 1915,   46,   44,   43,   42,   41,
- /*   740 */    98,  435, 2623,  530,  528,  770,  769, 2126, 2127, 2129,
- /*   750 */  2130, 2131, 2442,   12,  289,   10,  413,  439,  288,  517,
- /*   760 */  1788, 1791,  513,  509,  505,  502,  531, 2379, 1920, 1921,
- /*   770 */  1974, 1584, 1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985,
- /*   780 */  1986,  836,  857,  856, 1998, 2000, 2001, 2002, 2003,    2,
- /*   790 */    47,   45, 2009, 1498, 2705, 1497,  754,  480,  469,  236,
- /*   800 */  1916,  500,  180,  865, 2895,  889,  499,  841, 2900, 2852,
- /*   810 */  2664,  487, 2386, 2004, 1585, 1914,  322,  489, 2705,   78,
- /*   820 */  2437, 2439, 2901,  217, 1601, 1944, 2812, 2896,  789, 1499,
- /*   830 */   788,  841,  101, 2865,  636, 2723, 1941,  424, 2895, 1600,
- /*   840 */   451, 2444,  726,  634, 1999,   92,  630,  626,   91,  462,
- /*   850 */   211, 2670, 2807,  840,  829,  754,  787,  217, 1922, 2723,
- /*   860 */  2442, 2896,  789, 2895,  855, 2384,  896,  172,  171,  893,
- /*   870 */   892,  891,  169,   89,  474, 2670,  322,  840,  443,  442,
- /*   880 */   320, 2901,  217,  483,  539,  859, 2896,  789,  959, 2463,
- /*   890 */  2371,   48,  896,  172,  171,  893,  892,  891,  169, 2704,
- /*   900 */  1501, 1502, 2743,    3, 2705,  738,  119, 2706,  844, 2708,
- /*   910 */  2709,  839,  609,  859, 1916,   53,   90,  841, 2915, 2163,
- /*   920 */  2797,  855, 2384, 2704,  465, 2793, 2743, 2006, 2007, 1914,
- /*   930 */   119, 2706,  844, 2708, 2709,  839,  824,  859, 2769,  551,
- /*   940 */  2507,  540, 2915,  796, 2797, 2723,  557, 2507,  465, 2793,
- /*   950 */   472,  864,  863,  862,  855, 2384,  441,  440,  179,  687,
- /*   960 */  2444, 2670,  826,  840, 2769, 2705, 1977, 1987,  473, 2389,
- /*   970 */   855, 2384, 1922, 1478,  661, 1885, 2005, 2008,  841, 2442,
- /*   980 */   765,  689, 2366,  855, 2384,  688, 2444,  740, 2577,  297,
- /*   990 */   559, 1917,  232, 1915,  488,  320, 2120, 1476, 1477,  234,
- /*  1000 */   588, 2507,  959,  573, 1978, 2442, 2723,  479,  478, 2704,
- /*  1010 */    62, 2121, 2743, 1945,  681,  680,  187, 2706,  844, 2708,
- /*  1020 */  2709,  839, 2670,  859,  840, 2091, 2204, 1920, 1921, 1974,
- /*  1030 */  2025, 1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986,
- /*  1040 */   836,  857,  856, 1998, 2000, 2001, 2002, 2003,    2,   47,
- /*  1050 */    45, 2705,  490,  239, 2119,  581,  580,  469,  382, 1916,
- /*  1060 */   179, 1884,  683,  682,  841,  102, 2888, 1975,  855, 2384,
- /*  1070 */  2704, 2389, 2004, 2743, 1914,  790, 2916,  119, 2706,  844,
- /*  1080 */  2708, 2709,  839, 2670,  859,  702,  701, 2705,  574, 2915,
- /*  1090 */   182, 2797, 2723,  482,  481,  465, 2793, 1605, 2318, 1817,
- /*  1100 */   841, 2203, 2829, 1999, 1650, 1917, 2388, 1915, 2670,  533,
- /*  1110 */   840,  532, 1604,  111,   14,   13, 2202, 1922, 1641,  888,
- /*  1120 */   887,  886, 1645,  885, 1647, 1648,  884,  881, 2723, 1656,
- /*  1130 */   878, 1658, 1659,  875,  872,  869, 2195,  855, 2384, 2377,
- /*  1140 */  2319, 1920, 1921,  531, 2670,  117,  840,  959, 2705, 2899,
- /*  1150 */    48, 2035,  855, 2384,  855, 2384, 2704,  575, 2670, 2743,
- /*  1160 */  2359,  841,  158,  119, 2706,  844, 2708, 2709,  839,  376,
- /*  1170 */   859, 2376, 2381, 2670,  292, 2915, 2201, 2797,  855, 2384,
- /*  1180 */   190,  465, 2793,  212,  148,  611, 2006, 2007, 1480, 2723,
- /*  1190 */   855, 2384, 2704,  322, 1940, 2743,   88, 2200,  300,  119,
- /*  1200 */  2706,  844, 2708, 2709,  839, 2670,  859,  840,  855, 2384,
- /*  1210 */   806, 2915, 2199, 2797,  494,  493, 2198,  465, 2793, 2197,
- /*  1220 */   855, 2384,  855, 2384, 1930, 1977, 1987, 2047,  333,  855,
- /*  1230 */  2384, 2586,  898, 2670, 2858, 2005, 2008, 2004,   34, 1923,
- /*  1240 */   820, 1945,  340, 1941, 1946,   43,   42,   41, 2040,  852,
- /*  1250 */  1917, 2591, 1915, 2704, 2670, 2610, 2743,  855, 2384,  771,
- /*  1260 */   119, 2706,  844, 2708, 2709,  839, 2196,  859, 1999, 2670,
- /*  1270 */   926,  924, 2772, 2670, 2797, 2193, 2670,  853,  465, 2793,
- /*  1280 */   156,  689, 1922, 2768, 2192,  688, 1920, 1921, 1974, 2272,
- /*  1290 */  1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986,  836,
- /*  1300 */   857,  856, 1998, 2000, 2001, 2002, 2003,    2,   47,   45,
- /*  1310 */  2191, 2705,  828,  291,  855, 2384,  469, 2190, 1916,  855,
- /*  1320 */  2384, 2189, 2188, 2670,  841,  308,  179, 2624, 1650,  782,
- /*  1330 */  2187, 2004, 2670, 1914,  368,  890,  792, 2390, 2435,  491,
- /*  1340 */  2186, 2670, 1641,  888,  887,  886, 1645,  885, 1647, 1648,
- /*  1350 */   835,  834, 2723, 1656,  833, 1658, 1659,  832,  872,  869,
- /*  1360 */   894, 2185, 1999, 2435,  706,  705,  704, 2670, 2670,  170,
- /*  1370 */   840,  696,  152,  700, 2670,  316, 1922,  699, 2670, 2670,
- /*  1380 */  2444,  754,  698,  703,  446,  445, 2444, 2670,  697, 2895,
- /*  1390 */  2237, 2444,  444,  693,  692,  691, 2081, 2670,  514,  807,
- /*  1400 */   895,  800,  155, 2435, 2614,  815,  959, 2901,  217,   15,
- /*  1410 */  2443,  383, 2896,  789, 2421, 1931, 2704, 1926, 2670, 2743,
- /*  1420 */  2210,  954,  223,  119, 2706,  844, 2708, 2709,  839,  277,
- /*  1430 */   859,  279,  275,  281,  278, 2770,  280, 2797,  419,  418,
- /*  1440 */  2361,  465, 2793,  950,  283, 2006, 2007,  282,  475,  162,
- /*  1450 */    54, 1934, 1936,  162,  694,  695,  507,  728, 2691,  727,
- /*  1460 */   272, 2004, 2705,  484,  791,  857,  856, 1998, 2000, 2001,
- /*  1470 */  2002, 2003, 1946,  161, 1975,  841, 2256,  191, 1582, 1580,
- /*  1480 */  2254, 2172, 2173, 1562, 1977, 1987,  679,  675,  671,  667,
- /*  1490 */  2245,  271, 1999, 2243, 2005, 2008,  207,  801,  707,  150,
- /*  1500 */    40,   39,  709, 2723,   46,   44,   43,   42,   41, 1917,
- /*  1510 */   761, 1915,  711, 2724,   49,  714,  170,  906,   49, 2670,
- /*  1520 */   200,  840, 2693,  186, 2805, 2806, 1563,  153, 2810,  327,
- /*  1530 */  1833, 2311,   77, 2310, 1841,   64, 2516,   99,  347,  346,
- /*  1540 */   269, 1554,   14,   13,   49, 1920, 1921, 1974, 2226, 1976,
- /*  1550 */  1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986,  836,  857,
- /*  1560 */   856, 1998, 2000, 2001, 2002, 2003,    2, 2704,   49,  736,
- /*  1570 */  2743,   77,  349,  348,  119, 2706,  844, 2708, 2709,  839,
- /*  1580 */   167,  859,  731,  170,  351,  350,  825,  962, 2797,  353,
- /*  1590 */   352, 2115,  465, 2793, 2705, 2125,   74,  805,  867, 2124,
- /*  1600 */   168,  306, 2848,  374,  355,  354,  170,  841,  257,  768,
- /*  1610 */   331,  458,  775,  812,  357,  356, 2041,  268,  454,  952,
- /*  1620 */   206,  259,  266,  793,  149, 1988, 1924,  264,  653,  948,
- /*  1630 */   944,  940,  936,  167,  371, 2723,  754,  809,  498, 1907,
- /*  1640 */   516, 1883,  359,  358, 2895,  907,  256,  361,  360, 1786,
- /*  1650 */  2517, 2670, 1776,  840,  363,  362,  365,  364, 1535,  367,
- /*  1660 */   366,  343, 2901,  217, 1632, 2232,  747, 2896,  789, 1552,
- /*  1670 */  2432, 2705,  783,  477,  476, 1908, 2849,  381, 2859, 1663,
- /*  1680 */   118, 1671,  123,  344,  841,  784,  318, 1678,  713,  857,
- /*  1690 */   856, 1998, 2000, 2001, 2002, 2003,  313,  321, 2345, 2704,
- /*  1700 */     5, 1536, 2743,  725, 2705, 1676,  120, 2706,  844, 2708,
- /*  1710 */  2709,  839, 2723,  859,  173,  816,  501,  841,  506,  287,
- /*  1720 */  2797, 1939,  433, 1949, 2796, 2793,  515,  526, 2670,  527,
- /*  1730 */   840,  224,  529,  225, 2705, 1810,  716,  227,  375,  543,
- /*  1740 */  1940,  550,  238,  710,  708, 2723,  552,  841,  600,  556,
- /*  1750 */   284, 2705,  558,  563,  587,  576, 2509,  589,  613,  612,
- /*  1760 */   342, 2670,  822,  840,  841,  325,  596,  610,  599,  601,
- /*  1770 */   324,  243,  244,  615,  247, 2723, 2704,  617,  618, 2743,
- /*  1780 */   620,  637,  622,  120, 2706,  844, 2708, 2709,  839,  294,
- /*  1790 */   859, 2670, 2723,  840, 1947,    4,   72, 2797,  638,   71,
- /*  1800 */   646,  827, 2793,  645, 1927,  648,  255,   94, 2670,  842,
- /*  1810 */   840, 1942, 2743,  650,  258, 1948,  120, 2706,  844, 2708,
- /*  1820 */  2709,  839,  651,  859, 1950,  652,  739,  261, 2705,  804,
- /*  1830 */  2797,  654,  263, 1951,  428, 2793, 2532,   95, 1952, 2704,
- /*  1840 */   684,  841, 2743, 2526,   96, 2705,  184, 2706,  844, 2708,
- /*  1850 */  2709,  839,   97,  859,  663,  270, 2704,  410,  841, 2743,
- /*  1860 */   717,  718,  124,  120, 2706,  844, 2708, 2709,  839, 2723,
- /*  1870 */   859,  686, 2600, 2374,  274, 2597,  100, 2797, 2370,  276,
- /*  1880 */   754,  175, 2794,  754,  121, 2670, 2723,  840, 2895, 2372,
- /*  1890 */  2367, 2895,  176,  755, 2855,  177, 2596,  730,  377,  163,
- /*  1900 */   732,  293, 2670, 1943,  840, 2578, 2901,  217,  742, 2901,
- /*  1910 */   217, 2896,  789,  741, 2896,  789,  298,  743,  296,  749,
- /*  1920 */   746,  748,  758,  772,  810, 2705,    8,  456, 2864, 2863,
- /*  1930 */  2836,  303,  781, 2704,  307,  189, 2743,  759,  841,  757,
- /*  1940 */   184, 2706,  844, 2708, 2709,  839,  305,  859,  311,  309,
- /*  1950 */  2704,  756,  312, 2743, 2705,  310,  786,  406, 2706,  844,
- /*  1960 */  2708, 2709,  839,  459,  859,  785, 2723,  841, 2918,  794,
- /*  1970 */   797, 2816,  314,  154, 1944, 2894,  317, 2089, 2087,  203,
- /*  1980 */   323, 2813, 2670,  164,  840,  808,  378, 2546, 2856, 2545,
- /*  1990 */     1,  219, 2544,  379,  463, 2723,  813,  814,  818,  165,
- /*  2000 */    61,  336,  821, 2778,  846,  848,  850,  457,  341,  851,
- /*  2010 */   380, 2670,  110,  840, 2385, 2705, 2662,  112, 1455, 2661,
- /*  2020 */  2657,  953,  861, 2656, 2648,  384, 2647,  956,  841, 2639,
- /*  2030 */  2704, 2638, 2654, 2743, 2653, 2645,  370,  406, 2706,  844,
- /*  2040 */  2708, 2709,  839, 2644,  859, 2633,   52,  174, 2632, 2651,
- /*  2050 */   408, 2705,  958, 2650,  734, 2642, 2723,  396,  386, 2704,
- /*  2060 */  2641, 2630, 2743, 2629,  838, 2627,  399, 2706,  844, 2708,
- /*  2070 */  2709,  839, 2670,  859,  840,  420, 2626, 2436,  407,  397,
- /*  2080 */  2622,  388, 2621, 2620,  425,   83,  426, 2615,  503,  504,
- /*  2090 */  1867, 1868, 2723,  222, 2613,  508,  510,  511,  512, 1866,
- /*  2100 */  2612, 2611,  434, 2609,  518, 2608,  520, 2607, 2670, 2606,
- /*  2110 */   840,  522,  524, 1854, 2582,  226, 2581,  780,  228, 1812,
- /*  2120 */  2704,   84, 1813, 2743, 2559, 2558, 2557,  187, 2706,  844,
- /*  2130 */  2708, 2709,  839,  536,  859,  537, 2556, 2555, 2499,  541,
- /*  2140 */  2705, 1749, 2496,  544, 2495, 2489, 2486,  547,  548, 2485,
- /*  2150 */  2484,  231, 2483,  841,  235,  564, 2704,   87, 2488, 2743,
- /*  2160 */  2487, 2482,  233,  405, 2706,  844, 2708, 2709,  839, 2705,
- /*  2170 */   859, 2481, 2763, 2479, 2478, 2477, 2476,  566, 2474, 2473,
- /*  2180 */  2472, 2723,  841, 2471, 2470, 2494, 2469, 2917, 2468, 2467,
- /*  2190 */  2705, 1755,  237, 2456, 2455, 2454,   93, 2670, 2492,  840,
- /*  2200 */  2475, 2466, 2465,  841, 2464, 2462, 2461, 2460, 2459, 2458,
- /*  2210 */  2723, 2457, 2453, 2452, 2524, 2493, 2491, 2451, 2450, 2705,
- /*  2220 */  2449, 2448,  468, 2447,  242,  603, 2670, 2446,  840,  605,
- /*  2230 */  2445, 2723,  841, 2276,  245, 2275,  246,  422, 2274, 2273,
- /*  2240 */   423, 1602, 2271,  248, 1606, 2704, 2268, 2670, 2743,  840,
- /*  2250 */  2267,  470,  406, 2706,  844, 2708, 2709,  839,  249,  859,
- /*  2260 */  2723, 1598,  623, 2260, 2247, 2221,  627, 1479,  625,  631,
- /*  2270 */  2220,  635, 2580,  629, 2704,  262, 2670, 2743,  840,  253,
- /*  2280 */   624,  406, 2706,  844, 2708, 2709,  839,  633,  859,  628,
- /*  2290 */   198,   80,  632,  251, 2576,  729, 2690,   81, 2743, 2566,
- /*  2300 */   209, 2554,  401, 2706,  844, 2708, 2709,  839,  643,  859,
- /*  2310 */   260, 2553, 2530, 2705, 2523,  265,  267, 2362, 1528, 2270,
- /*  2320 */  2266,  664,  666,  665, 2704, 2264,  841, 2743,  668,  669,
- /*  2330 */  2705,  391, 2706,  844, 2708, 2709,  839,  670,  859, 2262,
- /*  2340 */   672,  673,  674,  841, 2259,  676,  678, 2705,  677, 2242,
- /*  2350 */  2240, 2241, 2239, 2217, 2723, 2364, 1682, 2363, 1683, 1588,
- /*  2360 */   841, 1587,   73, 1586, 1583, 1581, 1579, 1578, 1577, 1576,
- /*  2370 */  2670, 2723,  840,  273,  923, 1575,  925, 1572, 1570, 1571,
- /*  2380 */  2257,  447, 1569, 2255,  448, 2246,  449, 2670, 2723,  840,
- /*  2390 */  2244,  450, 2216, 2215, 2214,  715, 2213,  719, 2212,  723,
- /*  2400 */   712,  126,  721, 1848, 2670, 1850,  840, 1847, 2579, 2575,
- /*  2410 */  2705, 1852,  295, 1819, 2565,   28, 2552, 1823, 2704,   67,
- /*  2420 */  2551, 2743, 1838,  841,  733,  390, 2706,  844, 2708, 2709,
- /*  2430 */   839, 2900,  859,   56, 1821, 2704,   20,  744, 2743,   17,
- /*  2440 */    21,  760,  392, 2706,  844, 2708, 2709,  839,   57,  859,
- /*  2450 */   745, 2723, 2704, 2550,  455, 2743,    6,  299,  764,  398,
- /*  2460 */  2706,  844, 2708, 2709,  839,   22,  859, 2670, 1798,  840,
- /*  2470 */   202, 1797,  178,  750,  752,    7,  214,   32, 2691, 2082,
- /*  2480 */    30,  766,   23,  215, 2084, 2142,  302,   65, 2116,  304,
- /*  2490 */  2705,  762,   24, 2123, 2157,  188, 2529,  201, 2156,   31,
- /*  2500 */   460, 2110,   82,  841,   18, 2080, 2161, 2705,  216, 2160,
- /*  2510 */   461, 2162,   59,  319, 2163, 2704, 2064,  193, 2743, 2063,
- /*  2520 */   841, 2528,  402, 2706,  844, 2708, 2709,  839,  105,  859,
- /*  2530 */   106, 2723,  104, 2705, 2522,  326,  107, 2118,  817,  204,
- /*  2540 */    25,  811,   13,  332,   69, 2016,  841, 2670, 2723,  840,
- /*  2550 */  1932,  334,  335, 2015,   58,   11, 2026,  194,  205, 1991,
- /*  2560 */  1990,  874,  877, 1967, 2670,  880,  840,  883,   38,   16,
- /*  2570 */  2705, 2521, 1989,   26, 2723,  843, 1959,  108,   27,   70,
- /*  2580 */   339,  819,  345,  841,  847, 1993, 2178,  109, 2177, 2748,
- /*  2590 */  2670, 2747,  840,  337,  858, 2704,   68,  113, 2743, 2176,
- /*  2600 */   860, 1664,  393, 2706,  844, 2708, 2709,  839,  868,  859,
- /*  2610 */   866, 2723, 2704,  486, 1661, 2743,  870, 1660,  871,  403,
- /*  2620 */  2706,  844, 2708, 2709,  839,  873,  859, 2670,  849,  840,
- /*  2630 */  1657, 2705,  876, 1651,  879,  882, 2175, 1649, 2704,  114,
- /*  2640 */  1655, 2743,  369,  115,  841,  394, 2706,  844, 2708, 2709,
- /*  2650 */   839, 1654,  859, 1677, 2705,   79, 1653, 1673, 1652, 1526,
- /*  2660 */   897, 1566, 1565, 1564, 1561, 1558, 1557,  841, 1556, 1555,
- /*  2670 */  1553, 1551, 2723, 1550, 1549, 2704, 1596,  911, 2743, 1595,
- /*  2680 */   220,  913,  404, 2706,  844, 2708, 2709,  839, 2670,  859,
- /*  2690 */   840, 1547, 1546, 1545, 1544, 2723, 1543, 1542, 1541, 1592,
- /*  2700 */  1590, 1538, 1537, 1534, 1533, 1532, 1531, 2265,  933,  934,
- /*  2710 */   935, 2670, 2263,  840,  937, 2705,  939, 2261,  941, 2258,
- /*  2720 */   943,  945,  938, 2238,  947,  942,  949,  946,  841, 1468,
- /*  2730 */   951, 2211, 1456,  955, 2181,  373, 2704, 2236, 1918, 2743,
- /*  2740 */  2705,  957,  960,  395, 2706,  844, 2708, 2709,  839,  385,
- /*  2750 */   859, 2181,  961,  841, 2181, 2181, 2723, 2181, 2181, 2704,
- /*  2760 */  2181, 2181, 2743, 2181, 2181, 2181,  411, 2706,  844, 2708,
- /*  2770 */  2709,  839, 2670,  859,  840, 2181, 2181, 2181, 2181, 2181,
- /*  2780 */  2181, 2723, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  2790 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2670, 2181,  840,
- /*  2800 */  2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  2810 */  2181, 2181, 2181, 2181,  841, 2181, 2181, 2181, 2181, 2181,
- /*  2820 */  2704, 2181, 2705, 2743, 2181, 2181, 2181,  412, 2706,  844,
- /*  2830 */  2708, 2709,  839, 2181,  859,  841, 2181, 2181, 2181, 2181,
- /*  2840 */  2181, 2181, 2723, 2181, 2181, 2704, 2181, 2181, 2743, 2181,
- /*  2850 */  2181, 2181, 2717, 2706,  844, 2708, 2709,  839, 2670,  859,
- /*  2860 */   840, 2181, 2181, 2723, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  2870 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2670,
- /*  2880 */  2181,  840, 2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  2890 */  2181, 2181, 2181, 2181, 2181, 2181,  841, 2181, 2181, 2181,
- /*  2900 */  2181, 2181, 2181, 2181, 2181, 2181, 2704, 2181, 2181, 2743,
- /*  2910 */  2181, 2181, 2181, 2716, 2706,  844, 2708, 2709,  839, 2181,
- /*  2920 */   859, 2181, 2181, 2181, 2723, 2181, 2181, 2704, 2181, 2181,
- /*  2930 */  2743, 2181, 2181, 2181, 2715, 2706,  844, 2708, 2709,  839,
- /*  2940 */  2670,  859,  840, 2181, 2705, 2181, 2181, 2181, 2181, 2181,
- /*  2950 */  2181, 2181, 2181, 2181, 2181, 2181, 2181,  841, 2181, 2181,
- /*  2960 */  2181, 2181, 2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  2970 */  2181, 2181, 2181, 2181, 2181, 2181,  841, 2181, 2181, 2181,
- /*  2980 */  2181, 2181, 2181, 2181, 2181, 2723, 2181, 2181, 2704, 2181,
- /*  2990 */  2181, 2743, 2181, 2181, 2181,  430, 2706,  844, 2708, 2709,
- /*  3000 */   839, 2670,  859,  840, 2723, 2181, 2181, 2181, 2181, 2181,
- /*  3010 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  3020 */  2670, 2181,  840, 2181, 2705, 2181, 2181, 2181, 2181, 2181,
- /*  3030 */  2181, 2181, 2181, 2181, 2181, 2181, 2181,  841, 2181, 2181,
- /*  3040 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2704,
- /*  3050 */  2181, 2181, 2743, 2181, 2181, 2181,  431, 2706,  844, 2708,
- /*  3060 */  2709,  839, 2181,  859, 2181, 2723, 2181, 2181, 2704, 2181,
- /*  3070 */  2181, 2743, 2181, 2181, 2181,  427, 2706,  844, 2708, 2709,
- /*  3080 */   839, 2670,  859,  840, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  3090 */  2181, 2181, 2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  3100 */  2181, 2181, 2181, 2181, 2181, 2181,  841, 2181, 2705, 2181,
- /*  3110 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  3120 */  2181,  841, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2704,
- /*  3130 */  2181, 2181, 2743, 2181, 2723, 2181,  432, 2706,  844, 2708,
- /*  3140 */  2709,  839, 2181,  859, 2181, 2181, 2181, 2181, 2181, 2723,
- /*  3150 */  2670, 2181,  840, 2181, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  3160 */  2181, 2181, 2181, 2181, 2181, 2670, 2181,  840, 2181, 2181,
- /*  3170 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  3180 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181,
- /*  3190 */  2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181,  842, 2181,
- /*  3200 */  2181, 2743, 2181, 2181, 2181,  401, 2706,  844, 2708, 2709,
- /*  3210 */   839, 2181,  859, 2704, 2181, 2181, 2743, 2181, 2181, 2181,
- /*  3220 */   400, 2706,  844, 2708, 2709,  839, 2181,  859,
+ /*    80 */  1923, 2270,  802,  155, 2671,  210,  842,  726,  136, 2687,
+ /*    90 */  2690,  135,  134,  133,  132,  131,  130,  129,  128,  127,
+ /*   100 */   861, 2432,  724,  252,  722,  286,  285,  644, 2705, 2234,
+ /*   110 */   961, 2744, 2813,   15, 1943,  119, 2707,  846, 2709, 2710,
+ /*   120 */   841,  649,  861,  103,  642, 2226,  197,  199,  856, 2798,
+ /*   130 */   646, 2140, 2705,  467, 2794, 2744,  643,   62, 2809,  406,
+ /*   140 */  2707,  846, 2709, 2710,  841,  839,  861,  825, 2763, 2007,
+ /*   150 */  2008,   50,  439,  218, 2512,  755,  934,  933,  932,  931,
+ /*   160 */   499, 2845,  930,  929,  160,  924,  923,  922,  921,  920,
+ /*   170 */   919,  918,  159,  912,  911,  910,  498,  497,  907,  906,
+ /*   180 */   905,  196,  195,  904,  494,  903,  902,  901, 1978, 1988,
+ /*   190 */   804,  185, 2806, 2807,  780,  153, 2811,  290, 2006, 2009,
+ /*   200 */   802,  155, 1763, 1764,  192, 2806,  801,  581,  147,  800,
+ /*   210 */   790, 2664,  580, 1918,  487, 1916, 2896,    9, 2896, 2100,
+ /*   220 */   579, 1927,   40,   39,  739, 1942,   46,   44,   43,   42,
+ /*   230 */    41, 2361, 2706, 2206,  789,  217,  789,  217,  122, 2897,
+ /*   240 */   791, 2897,  791, 1690, 1691,  843, 2389,  489, 2387, 1921,
+ /*   250 */  1922, 1975,  779, 1977, 1980, 1981, 1982, 1983, 1984, 1985,
+ /*   260 */  1986, 1987,  838,  859,  858, 1999, 2001, 2002, 2003, 2004,
+ /*   270 */     2,   47,   45, 2724,  917,  476,  415, 2345, 1940,  471,
+ /*   280 */  2724, 1917, 1844, 1845, 2901,  588,  861,  431,  609, 2671,
+ /*   290 */  2671,  842, 2896,  608, 2005,   60, 1915,   40,   39,  455,
+ /*   300 */  2586,   46,   44,   43,   42,   41,  753, 2015,  736, 1946,
+ /*   310 */   567, 2900,  610, 1942,  197, 2897, 2899,  416,  569, 1499,
+ /*   320 */   322, 1498,  125, 2806, 2807, 2000,  153, 2811,  180,  547,
+ /*   330 */   857, 2385,   19,  664,  322, 2034,  151, 2705, 2388, 1923,
+ /*   340 */  2744, 2071, 2513,  775,  119, 2707,  846, 2709, 2710,  841,
+ /*   350 */   221,  861, 1472,  778,  157, 1500,  166, 2769, 2798, 2183,
+ /*   360 */    40,   39,  467, 2794,   46,   44,   43,   42,   41,  961,
+ /*   370 */   440, 1479,   15,  898,  172,  171,  895,  894,  893,  169,
+ /*   380 */  1946,  555, 1942,  145,  144,  143,  142,  141,  140,  139,
+ /*   390 */   138,  137,  657, 1979, 1474, 1477, 1478,  537,   62,   40,
+ /*   400 */    39, 2035,  536,   46,   44,   43,   42,   41, 2007, 2008,
+ /*   410 */   661, 2519, 2498,  660,  596,  595,  594,  593,  592,  587,
+ /*   420 */   586,  585,  584,  423,   29, 1975, 2071,  574,  573,  572,
+ /*   430 */   571,  570,  564,  563,  562,  254,  557,  556,  438,  644,
+ /*   440 */   856, 2234,  548, 1751, 1752,  329,  330, 1978, 1988, 1770,
+ /*   450 */   328,  756,  781,  776,  769,  765, 1976, 2006, 2009, 2896,
+ /*   460 */  2818, 2068, 2069, 2070, 2818, 2818, 2818, 2818, 2818, 2526,
+ /*   470 */  2376,  662, 1918,  521, 1916,   62,  900, 2902,  217,  213,
+ /*   480 */   740, 1979, 2897,  791,  857, 2385,  857, 2385,   36,  469,
+ /*   490 */  2029, 2030, 2031, 2032, 2033, 2037, 2038, 2039, 2040, 2073,
+ /*   500 */  2074, 2075, 2076, 2077,  146, 2141,   55,  377, 1921, 1922,
+ /*   510 */  1975,  687, 1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986,
+ /*   520 */  1987,  838,  859,  858, 1999, 2001, 2002, 2003, 2004,    2,
+ /*   530 */    12,   47,   45,   50,  476, 1926, 2706, 2539,  181,  471,
+ /*   540 */  1947, 1917,  116,  390, 1976,  861, 2068, 2069, 2070,  805,
+ /*   550 */   208,  113,  473,  600, 2005, 1590, 1915, 2171, 2536,  847,
+ /*   560 */   388,   76,  651, 2578,   75, 2706,   40,   39,  454, 2586,
+ /*   570 */    46,   44,   43,   42,   41,  417,  737, 2724,  843, 2071,
+ /*   580 */  2236,  322, 1943, 1923, 2681, 2000, 2901,  250,  623,  621,
+ /*   590 */   618,  616,   19, 2671, 2896,  842,   40,   39, 1592, 1923,
+ /*   600 */    46,   44,   43,   42,   41, 2491, 2724,  708,  707,  706,
+ /*   610 */   525, 1947, 2685, 2900,  698,  152,  702, 2897, 2898,  241,
+ /*   620 */   701, 2539, 2671,  658,  842,  700,  705,  448,  447,  961,
+ /*   630 */   914,  699,   15,   62,   98,  446,  695,  694,  693,  527,
+ /*   640 */   523, 2705, 2537,  847, 2744,  599,  240,  551,  119, 2707,
+ /*   650 */   846, 2709, 2710,  841,  797,  861,  857, 2385,  322,  597,
+ /*   660 */   199, 2381, 2798, 2687, 2689,  468,  467, 2794, 2007, 2008,
+ /*   670 */  2705,   63, 2112, 2744,  861, 2170,  146,  119, 2707,  846,
+ /*   680 */  2709, 2710,  841,  692,  861,   12, 2129,  659, 2532, 2916,
+ /*   690 */   183, 2798, 2195,   51, 2846,  467, 2794,  916, 2818, 2068,
+ /*   700 */  2069, 2070, 2818, 2818, 2818, 2818, 2818, 1978, 1988,   35,
+ /*   710 */  1945, 1790, 1791, 1929, 2181,   40,   39, 2006, 2009,   46,
+ /*   720 */    44,   43,   42,   41, 2359,   86,   85,  540, 2439, 2440,
+ /*   730 */   229, 2093, 1918, 2445, 1916,   46,   44,   43,   42,   41,
+ /*   740 */    98,  437, 2624,  532,  530,  772,  771, 2127, 2128, 2130,
+ /*   750 */  2131, 2132, 2443,   12,  289,   10,  414,  441,  288,  519,
+ /*   760 */  1789, 1792,  515,  511,  507,  504,  533, 2380, 1921, 1922,
+ /*   770 */  1975, 1585, 1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986,
+ /*   780 */  1987,  838,  859,  858, 1999, 2001, 2002, 2003, 2004,    2,
+ /*   790 */    47,   45, 2010, 1499, 2706, 1498,  756,  482,  471,  236,
+ /*   800 */  1917,  502,  180,  867, 2896,  891,  501,  843, 2901, 2853,
+ /*   810 */  2665,  489, 2387, 2005, 1586, 1915,  322,  491, 2706,   78,
+ /*   820 */  2438, 2440, 2902,  217, 1602, 1945, 2813, 2897,  791, 1500,
+ /*   830 */   790,  843,  101, 2866,  638, 2724, 1942,  426, 2896, 1601,
+ /*   840 */   453, 2445,  728,  636, 2000,   92,  632,  628,   91,  464,
+ /*   850 */   211, 2671, 2808,  842,  831,  756,  789,  217, 1923, 2724,
+ /*   860 */  2443, 2897,  791, 2896,  857, 2385,  898,  172,  171,  895,
+ /*   870 */   894,  893,  169,   89,  476, 2671,  322,  842,  445,  444,
+ /*   880 */   320, 2902,  217,  485,  541,  861, 2897,  791,  961, 2464,
+ /*   890 */  2372,   48,  898,  172,  171,  895,  894,  893,  169, 2705,
+ /*   900 */  1502, 1503, 2744,    3, 2706,  740,  119, 2707,  846, 2709,
+ /*   910 */  2710,  841,  611,  861, 1917,   53,   90,  843, 2916, 2164,
+ /*   920 */  2798,  857, 2385, 2705,  467, 2794, 2744, 2007, 2008, 1915,
+ /*   930 */   119, 2707,  846, 2709, 2710,  841,  826,  861, 2770,  553,
+ /*   940 */  2508,  542, 2916,  798, 2798, 2724,  559, 2508,  467, 2794,
+ /*   950 */   474,  866,  865,  864,  857, 2385,  443,  442,  179,  689,
+ /*   960 */  2445, 2671,  828,  842, 2770, 2706, 1978, 1988,  475, 2390,
+ /*   970 */   857, 2385, 1923, 1479,  663, 1886, 2006, 2009,  843, 2443,
+ /*   980 */   767,  691,  376,  857, 2385,  690, 2445,  742, 2578,  297,
+ /*   990 */   561, 1918,  232, 1916,  490,  320, 2121, 1477, 1478,  234,
+ /*  1000 */   590, 2508,  961,  575, 1979, 2443, 2724,  481,  480, 2705,
+ /*  1010 */    62, 2122, 2744, 1946,  683,  682,  187, 2707,  846, 2709,
+ /*  1020 */  2710,  841, 2671,  861,  842, 2092, 2205, 1921, 1922, 1975,
+ /*  1030 */  2026, 1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987,
+ /*  1040 */   838,  859,  858, 1999, 2001, 2002, 2003, 2004,    2,   47,
+ /*  1050 */    45, 2706,  492,  239, 2120,  583,  582,  471,  383, 1917,
+ /*  1060 */   179, 1885,  685,  684,  843,  102, 2889, 1976,  857, 2385,
+ /*  1070 */  2705, 2390, 2005, 2744, 1915,  792, 2917,  119, 2707,  846,
+ /*  1080 */  2709, 2710,  841, 2671,  861,  704,  703, 2706,  576, 2916,
+ /*  1090 */   182, 2798, 2724,  484,  483,  467, 2794, 1606, 2319, 1818,
+ /*  1100 */   843, 2204, 2830, 2000, 1651, 1918, 2389, 1916, 2671,  535,
+ /*  1110 */   842,  534, 1605,  111,   14,   13, 2203, 1923, 1642,  890,
+ /*  1120 */   889,  888, 1646,  887, 1648, 1649,  886,  883, 2724, 1657,
+ /*  1130 */   880, 1659, 1660,  877,  874,  871, 2196,  857, 2385, 2378,
+ /*  1140 */  2320, 1921, 1922,  533, 2671,  117,  842,  961, 2706, 2900,
+ /*  1150 */    48, 2036,  857, 2385,  857, 2385, 2705,  577, 2671, 2744,
+ /*  1160 */  2360,  843,  158,  119, 2707,  846, 2709, 2710,  841,  377,
+ /*  1170 */   861, 2377, 2382, 2671,  292, 2916, 2202, 2798,  857, 2385,
+ /*  1180 */   190,  467, 2794,  212,  148,  613, 2007, 2008, 1481, 2724,
+ /*  1190 */   857, 2385, 2705,  322, 1941, 2744,   88, 2201,  300,  119,
+ /*  1200 */  2707,  846, 2709, 2710,  841, 2671,  861,  842,  857, 2385,
+ /*  1210 */   808, 2916, 2200, 2798,  496,  495, 2199,  467, 2794, 2198,
+ /*  1220 */   857, 2385,  857, 2385, 1931, 1978, 1988, 2048,  333,  857,
+ /*  1230 */  2385, 2587,  900, 2671, 2859, 2006, 2009, 2005,   34, 1924,
+ /*  1240 */   822, 1946,  340, 1942, 1947,   43,   42,   41, 2041,  854,
+ /*  1250 */  1918, 2592, 1916, 2705, 2671, 2611, 2744,  857, 2385,  773,
+ /*  1260 */   119, 2707,  846, 2709, 2710,  841, 2197,  861, 2000, 2671,
+ /*  1270 */   928,  926, 2773, 2671, 2798, 2194, 2671,  855,  467, 2794,
+ /*  1280 */   156,  691, 1923, 2769, 2193,  690, 1921, 1922, 1975, 2273,
+ /*  1290 */  1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987,  838,
+ /*  1300 */   859,  858, 1999, 2001, 2002, 2003, 2004,    2,   47,   45,
+ /*  1310 */  2192, 2706,  830,  291,  857, 2385,  471, 2191, 1917,  857,
+ /*  1320 */  2385, 2190, 2189, 2671,  843,  308,  179, 2625, 1651,  784,
+ /*  1330 */  2188, 2005, 2671, 1915,  368,  892,  794, 2391, 2436,  493,
+ /*  1340 */  2187, 2671, 1642,  890,  889,  888, 1646,  887, 1648, 1649,
+ /*  1350 */   837,  836, 2724, 1657,  835, 1659, 1660,  834,  874,  871,
+ /*  1360 */   896, 2186, 2000, 2436,  708,  707,  706, 2671, 2671,  170,
+ /*  1370 */   842,  698,  152,  702, 2671,  316, 1923,  701, 2671, 2671,
+ /*  1380 */  2445,  756,  700,  705,  448,  447, 2445, 2671,  699, 2896,
+ /*  1390 */  2238, 2445,  446,  695,  694,  693, 2082, 2671,  516,  809,
+ /*  1400 */   897,  802,  155, 2436, 2615,  817,  961, 2902,  217,   15,
+ /*  1410 */  2444,  384, 2897,  791, 2422, 1932, 2705, 1927, 2671, 2744,
+ /*  1420 */  2211,  956,  223,  119, 2707,  846, 2709, 2710,  841,  277,
+ /*  1430 */   861,  279,  275,  281,  278, 2771,  280, 2798,  421,  420,
+ /*  1440 */  2362,  467, 2794,  952,  283, 2007, 2008,  282,  477,  162,
+ /*  1450 */    54, 1935, 1937,  162,  696,  697,  509,  730, 2692,  729,
+ /*  1460 */   272, 2005, 2706,  486,  793,  859,  858, 1999, 2001, 2002,
+ /*  1470 */  2003, 2004, 1947,  161, 1976,  843, 2257,  191, 1583, 1581,
+ /*  1480 */  2255, 2173, 2174, 1563, 1978, 1988,  681,  677,  673,  669,
+ /*  1490 */  2246,  271, 2000, 2244, 2006, 2009,  207,  803,  709,  150,
+ /*  1500 */    40,   39,  711, 2724,   46,   44,   43,   42,   41, 1918,
+ /*  1510 */   763, 1916,  713, 2725,   49,  716,  170,  908,   49, 2671,
+ /*  1520 */   200,  842, 2694,  186, 2806, 2807, 1564,  153, 2811,  327,
+ /*  1530 */  1834, 2312,   77, 2311, 1842,   64, 2517,   99,  347,  346,
+ /*  1540 */   269, 1555,   14,   13,   49, 1921, 1922, 1975, 2227, 1977,
+ /*  1550 */  1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987,  838,  859,
+ /*  1560 */   858, 1999, 2001, 2002, 2003, 2004,    2, 2705,   49,  738,
+ /*  1570 */  2744,   77,  349,  348,  119, 2707,  846, 2709, 2710,  841,
+ /*  1580 */   167,  861,  733,  170,  351,  350,  827,  964, 2798,  353,
+ /*  1590 */   352, 2116,  467, 2794, 2706, 2126,   74,  807,  869, 2125,
+ /*  1600 */   168,  306, 2849,  374,  355,  354,  170,  843,  257,  770,
+ /*  1610 */   331,  460,  777,  814,  357,  356, 2042,  268,  456,  954,
+ /*  1620 */   206,  259,  266,  795,  149, 1989, 1925,  264,  655,  950,
+ /*  1630 */   946,  942,  938,  167,  371, 2724,  756,  811,  500, 1908,
+ /*  1640 */   518, 1884,  359,  358, 2896,  909,  256,  361,  360, 1787,
+ /*  1650 */  2518, 2671, 1777,  842,  363,  362,  365,  364, 1536,  367,
+ /*  1660 */   366,  343, 2902,  217, 1633, 2233,  749, 2897,  791, 1553,
+ /*  1670 */  2433, 2706,  785,  479,  478, 1909, 2850,  382, 2860, 1664,
+ /*  1680 */   118, 1672,  123,  344,  843,  786,  318, 1679,  715,  859,
+ /*  1690 */   858, 1999, 2001, 2002, 2003, 2004,  313,  321, 2346, 2705,
+ /*  1700 */     5, 1537, 2744,  727, 2706, 1677,  120, 2707,  846, 2709,
+ /*  1710 */  2710,  841, 2724,  861,  173,  818,  503,  843,  508,  287,
+ /*  1720 */  2798, 1940,  435, 1950, 2797, 2794,  517,  528, 2671,  529,
+ /*  1730 */   842,  224,  531,  225, 2706, 1811,  718,  227,  375,  545,
+ /*  1740 */  1941,  552,  238,  712,  710, 2724,  554,  843,  602,  558,
+ /*  1750 */   284, 2706,  560,  565,  589,  578, 2510,  591,  615,  614,
+ /*  1760 */   342, 2671,  824,  842,  843,  325,  598,  612,  601,  603,
+ /*  1770 */   324,  243,  244,  617,  247, 2724, 2705,  619,  620, 2744,
+ /*  1780 */   622,  639,  624,  120, 2707,  846, 2709, 2710,  841,  294,
+ /*  1790 */   861, 2671, 2724,  842, 1948,    4,   72, 2798,  640,   71,
+ /*  1800 */   648,  829, 2794,  647, 1928,  650,  255,   94, 2671,  844,
+ /*  1810 */   842, 1943, 2744,  652,  258, 1949,  120, 2707,  846, 2709,
+ /*  1820 */  2710,  841,  653,  861, 1951,  654,  741,  261, 2706,  806,
+ /*  1830 */  2798,  656,  263, 1952,  430, 2794, 2533,   95, 1953, 2705,
+ /*  1840 */   686,  843, 2744, 2527,   96, 2706,  184, 2707,  846, 2709,
+ /*  1850 */  2710,  841,   97,  861,  665,  270, 2705,  411,  843, 2744,
+ /*  1860 */   719,  720,  124,  120, 2707,  846, 2709, 2710,  841, 2724,
+ /*  1870 */   861,  688, 2601, 2375,  274, 2598,  100, 2798, 2371,  276,
+ /*  1880 */   756,  175, 2795,  756,  121, 2671, 2724,  842, 2896, 2373,
+ /*  1890 */  2368, 2896,  176,  757, 2856,  177, 2597,  732,  378,  163,
+ /*  1900 */   734,  293, 2671, 1944,  842, 2579, 2902,  217,  744, 2902,
+ /*  1910 */   217, 2897,  791,  743, 2897,  791,  298,  745,  296,  751,
+ /*  1920 */   748,  750,  760,  774,  812, 2706,    8,  458, 2865, 2864,
+ /*  1930 */  2837,  303,  783, 2705,  307,  189, 2744,  761,  843,  759,
+ /*  1940 */   184, 2707,  846, 2709, 2710,  841,  305,  861,  311,  309,
+ /*  1950 */  2705,  758,  312, 2744, 2706,  310,  788,  407, 2707,  846,
+ /*  1960 */  2709, 2710,  841,  461,  861,  787, 2724,  843, 2919,  796,
+ /*  1970 */   799, 2817,  314,  154, 1945, 2895,  317, 2090, 2088,  203,
+ /*  1980 */   323, 2814, 2671,  164,  842,  810,  379, 2547, 2857, 2546,
+ /*  1990 */     1,  219, 2545,  380,  465, 2724,  815,  816,  820,  165,
+ /*  2000 */    61,  336,  823, 2779,  848,  850,  852,  459,  341,  853,
+ /*  2010 */   381, 2671,  110,  842, 2386, 2706, 2663,  112, 1456, 2662,
+ /*  2020 */  2658,  955,  863, 2657, 2649,  385, 2648,  958,  843, 2640,
+ /*  2030 */  2705, 2639, 2655, 2744, 2654, 2646,  370,  407, 2707,  846,
+ /*  2040 */  2709, 2710,  841, 2645,  861, 2634,   52,  174, 2633, 2652,
+ /*  2050 */   419, 2706,  960, 2651,  736, 2643, 2724,  397,  387, 2705,
+ /*  2060 */  2642, 2631, 2744, 2630,  840, 2628,  400, 2707,  846, 2709,
+ /*  2070 */  2710,  841, 2671,  861,  842,  422, 2627, 2437,  408,  398,
+ /*  2080 */   409,  389, 2623, 2622,  427, 2621,   83, 2616,  505,  428,
+ /*  2090 */   506, 1868, 2724, 1869,  222,  510, 2614,  512,  513,  514,
+ /*  2100 */  1867, 2613, 2612,  436, 2610,  520, 2609,  522, 2671, 2608,
+ /*  2110 */   842,  524, 2607,  526, 1855, 2583,  226,  782, 2582,  228,
+ /*  2120 */  2705,   84, 1814, 2744, 1813, 2560, 2559,  187, 2707,  846,
+ /*  2130 */  2709, 2710,  841, 2558,  861,  538,  539, 2557, 2556, 2500,
+ /*  2140 */  2706,  543, 1750, 2497,  546, 2496, 2490,  549, 2487,  550,
+ /*  2150 */  2486, 2485,  231,  843,  233, 2488, 2705, 2484,   87, 2744,
+ /*  2160 */  2489, 2483, 2482,  406, 2707,  846, 2709, 2710,  841, 2706,
+ /*  2170 */   861, 2480, 2764, 2479, 2478,  235,  566, 2477,  568, 2475,
+ /*  2180 */  2474, 2724,  843, 2473, 2472, 2471, 2495, 2918, 2470, 2469,
+ /*  2190 */  2706,  237, 2457, 2456,   93, 2455, 2454, 2671, 2468,  842,
+ /*  2200 */  2493, 2476, 2467,  843, 2466, 2465, 2463, 2462, 2461, 2460,
+ /*  2210 */  2724, 2459, 2458, 2453, 2525, 2494, 2492, 2452, 2451, 2706,
+ /*  2220 */  2450, 1756,  470,  242, 2449,  605, 2671, 2448,  842,  607,
+ /*  2230 */  2447, 2724,  843, 2446, 2277,  245, 2276,  424,  246, 2275,
+ /*  2240 */   425, 1603,  248, 1607, 2274, 2705, 2272, 2671, 2744,  842,
+ /*  2250 */  2269,  472,  407, 2707,  846, 2709, 2710,  841, 2268,  861,
+ /*  2260 */  2724, 2261, 1599,  249,  627,  625,  629, 2248,  631,  633,
+ /*  2270 */  2222, 1480, 2221,  635, 2705,  626, 2671, 2744,  842,  637,
+ /*  2280 */    80,  407, 2707,  846, 2709, 2710,  841,  630,  861,  634,
+ /*  2290 */   251,   81,  198, 2581, 2577,  731, 2691,  209, 2744,  253,
+ /*  2300 */   645, 2567,  402, 2707,  846, 2709, 2710,  841, 2555,  861,
+ /*  2310 */   260, 2554,  262, 2706,  265, 2531, 2524, 2363,  267, 2271,
+ /*  2320 */  2267,  666,  668,  667, 2705, 2265,  843, 2744, 1529,  670,
+ /*  2330 */  2706,  392, 2707,  846, 2709, 2710,  841,  671,  861,  672,
+ /*  2340 */  2263,  674,  675,  843,  676, 2260,  678, 2706,  679,  680,
+ /*  2350 */  2243, 2241, 2242, 2240, 2724, 2218, 2365, 1683, 1684, 2364,
+ /*  2360 */   843,   73,  273, 1589, 1588, 1587, 1584, 1582, 1580, 1579,
+ /*  2370 */  2671, 2724,  842, 2258, 1578, 1577, 1576,  925, 2256,  927,
+ /*  2380 */  1573, 2247, 1571, 2245, 1572, 1570,  449, 2671, 2724,  842,
+ /*  2390 */   450,  451,  452, 2217,  717, 2216,  714, 2215, 2214,  721,
+ /*  2400 */   723, 2213,  725, 1849, 2671,  126,  842, 2580, 1851, 1848,
+ /*  2410 */  2706, 2576, 1853,   28,  295, 2566,   67, 2553, 2705, 1820,
+ /*  2420 */  1839, 2744, 1822,  843,  746,  391, 2707,  846, 2709, 2710,
+ /*  2430 */   841,   56,  861, 2552, 2901, 2705,   20,   17, 2744,   30,
+ /*  2440 */     6,  762,  393, 2707,  846, 2709, 2710,  841, 2143,  861,
+ /*  2450 */    57, 2724, 2705,  457,  747, 2744, 1799,   21,  299,  399,
+ /*  2460 */  2707,  846, 2709, 2710,  841,  766,  861, 2671, 1798,  842,
+ /*  2470 */     7,  302,   22,  752,  754, 1824,  214,  202,  735,  178,
+ /*  2480 */    32, 2117,   23, 2692,  768, 2083,  215,  764, 2085,  188,
+ /*  2490 */  2706,  304,   65, 2124,   24, 2158, 2157,  201,  319,   31,
+ /*  2500 */   462, 2111,   82,  843,   18, 2081, 2163, 2706, 2164,  216,
+ /*  2510 */  2162, 2161,  463, 2065,   59, 2705,  193, 2551, 2744, 2530,
+ /*  2520 */   843,  104,  403, 2707,  846, 2709, 2710,  841, 2064,  861,
+ /*  2530 */   105, 2724,  326, 2706, 2529, 2119,  204,  106,  332,   69,
+ /*  2540 */  2523,  335,  813,  819,  107,   25,  843, 2671, 2724,  842,
+ /*  2550 */    13, 2017, 2016,   11, 1933, 2027, 1992,  194,  205, 1991,
+ /*  2560 */  1990,   58, 1960,  876, 2671,  879,  842,  882,  885,   38,
+ /*  2570 */  2706, 1968, 2522,   16, 2724,   26,  845,   27,   70,  108,
+ /*  2580 */   849,  339,  109,  843,  345,  334, 1994,  868, 2179, 2749,
+ /*  2590 */  2671, 2748,  842,  113, 1665, 2705,  860,   68, 2744,  821,
+ /*  2600 */   862,  337,  394, 2707,  846, 2709, 2710,  841, 2178,  861,
+ /*  2610 */   488, 2724, 2705, 2177,  870, 2744,  872, 1662,  873,  404,
+ /*  2620 */  2707,  846, 2709, 2710,  841,  851,  861, 2671, 1661,  842,
+ /*  2630 */   875, 2706, 1658,  878, 1652,  881, 2176, 1650, 2705,  884,
+ /*  2640 */  1656, 2744,  114,  369,  843,  395, 2707,  846, 2709, 2710,
+ /*  2650 */   841,  115,  861, 1678, 2706, 1655, 1654,   79, 1653, 1674,
+ /*  2660 */  1567, 1527,  899, 1566, 1565, 1562, 1559,  843, 1558, 1557,
+ /*  2670 */  1556, 1554, 2724, 1552, 1597, 2705, 1551, 1550, 2744, 1596,
+ /*  2680 */   220,  913,  405, 2707,  846, 2709, 2710,  841, 2671,  861,
+ /*  2690 */   842, 1548,  915, 1547, 1546, 2724, 1545, 1544, 1543, 1542,
+ /*  2700 */  1593, 1591, 1539, 1538, 1535, 1534, 1533, 1532, 2266,  935,
+ /*  2710 */   936, 2671, 2264,  842,  939, 2706, 2262,  937,  941,  943,
+ /*  2720 */   945,  940, 2259,  944,  947,  948, 2239,  949,  843,  951,
+ /*  2730 */  2237,  953, 1469, 2212,  957, 1457, 2705,  373,  959, 2744,
+ /*  2740 */  2706, 1919,  386,  396, 2707,  846, 2709, 2710,  841,  962,
+ /*  2750 */   861,  963, 2182,  843, 2182, 2182, 2724, 2182, 2182, 2705,
+ /*  2760 */  2182, 2182, 2744, 2182, 2182, 2182,  412, 2707,  846, 2709,
+ /*  2770 */  2710,  841, 2671,  861,  842, 2182, 2182, 2182, 2182, 2182,
+ /*  2780 */  2182, 2724, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  2790 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2671, 2182,  842,
+ /*  2800 */  2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  2810 */  2182, 2182, 2182, 2182,  843, 2182, 2182, 2182, 2182, 2182,
+ /*  2820 */  2705, 2182, 2706, 2744, 2182, 2182, 2182,  413, 2707,  846,
+ /*  2830 */  2709, 2710,  841, 2182,  861,  843, 2182, 2182, 2182, 2182,
+ /*  2840 */  2182, 2182, 2724, 2182, 2182, 2705, 2182, 2182, 2744, 2182,
+ /*  2850 */  2182, 2182, 2718, 2707,  846, 2709, 2710,  841, 2671,  861,
+ /*  2860 */   842, 2182, 2182, 2724, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  2870 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2671,
+ /*  2880 */  2182,  842, 2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  2890 */  2182, 2182, 2182, 2182, 2182, 2182,  843, 2182, 2182, 2182,
+ /*  2900 */  2182, 2182, 2182, 2182, 2182, 2182, 2705, 2182, 2182, 2744,
+ /*  2910 */  2182, 2182, 2182, 2717, 2707,  846, 2709, 2710,  841, 2182,
+ /*  2920 */   861, 2182, 2182, 2182, 2724, 2182, 2182, 2705, 2182, 2182,
+ /*  2930 */  2744, 2182, 2182, 2182, 2716, 2707,  846, 2709, 2710,  841,
+ /*  2940 */  2671,  861,  842, 2182, 2706, 2182, 2182, 2182, 2182, 2182,
+ /*  2950 */  2182, 2182, 2182, 2182, 2182, 2182, 2182,  843, 2182, 2182,
+ /*  2960 */  2182, 2182, 2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  2970 */  2182, 2182, 2182, 2182, 2182, 2182,  843, 2182, 2182, 2182,
+ /*  2980 */  2182, 2182, 2182, 2182, 2182, 2724, 2182, 2182, 2705, 2182,
+ /*  2990 */  2182, 2744, 2182, 2182, 2182,  432, 2707,  846, 2709, 2710,
+ /*  3000 */   841, 2671,  861,  842, 2724, 2182, 2182, 2182, 2182, 2182,
+ /*  3010 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  3020 */  2671, 2182,  842, 2182, 2706, 2182, 2182, 2182, 2182, 2182,
+ /*  3030 */  2182, 2182, 2182, 2182, 2182, 2182, 2182,  843, 2182, 2182,
+ /*  3040 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2705,
+ /*  3050 */  2182, 2182, 2744, 2182, 2182, 2182,  433, 2707,  846, 2709,
+ /*  3060 */  2710,  841, 2182,  861, 2182, 2724, 2182, 2182, 2705, 2182,
+ /*  3070 */  2182, 2744, 2182, 2182, 2182,  429, 2707,  846, 2709, 2710,
+ /*  3080 */   841, 2671,  861,  842, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  3090 */  2182, 2182, 2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  3100 */  2182, 2182, 2182, 2182, 2182, 2182,  843, 2182, 2706, 2182,
+ /*  3110 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  3120 */  2182,  843, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2705,
+ /*  3130 */  2182, 2182, 2744, 2182, 2724, 2182,  434, 2707,  846, 2709,
+ /*  3140 */  2710,  841, 2182,  861, 2182, 2182, 2182, 2182, 2182, 2724,
+ /*  3150 */  2671, 2182,  842, 2182, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  3160 */  2182, 2182, 2182, 2182, 2182, 2671, 2182,  842, 2182, 2182,
+ /*  3170 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  3180 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182,
+ /*  3190 */  2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182,  844, 2182,
+ /*  3200 */  2182, 2744, 2182, 2182, 2182,  402, 2707,  846, 2709, 2710,
+ /*  3210 */   841, 2182,  861, 2705, 2182, 2182, 2744, 2182, 2182, 2182,
+ /*  3220 */   401, 2707,  846, 2709, 2710,  841, 2182,  861,
 };
 static const YYCODETYPE yy_lookahead[] = {
  /*     0 */   506,  507,  438,  379,  386,  391,  421,  389,  390,  519,
@@ -1127,77 +1130,77 @@ static const YYCODETYPE yy_lookahead[] = {
  /*  2020 */   436,   38,  425,  436,  436,  391,  436,  381,  392,  436,
  /*  2030 */   484,  436,  436,  487,  436,  436,  398,  491,  492,  493,
  /*  2040 */   494,  495,  496,  436,  498,  436,  467,  385,  436,  436,
- /*  2050 */   476,  379,  384,  436,  483,  436,  420,  414,  399,  484,
+ /*  2050 */   422,  379,  384,  436,  483,  436,  420,  414,  399,  484,
  /*  2060 */   436,  436,  487,  436,  392,  436,  491,  492,  493,  494,
  /*  2070 */   495,  496,  436,  498,  438,  422,  436,  436,  414,  414,
- /*  2080 */     0,  377,    0,    0,  458,   45,  458,    0,   37,  240,
- /*  2090 */    37,   37,  420,   37,    0,  240,   37,   37,  240,   37,
- /*  2100 */     0,    0,  240,    0,   37,    0,   37,    0,  436,    0,
- /*  2110 */   438,   22,   37,  235,    0,  221,    0,  542,  221,  213,
- /*  2120 */   484,  222,  215,  487,    0,    0,    0,  491,  492,  493,
- /*  2130 */   494,  495,  496,  209,  498,  208,    0,    0,  154,   49,
- /*  2140 */   379,   49,    0,   37,    0,    0,    0,   37,   52,    0,
- /*  2150 */     0,   49,    0,  392,  173,   37,  484,   45,    0,  487,
- /*  2160 */     0,    0,   49,  491,  492,  493,  494,  495,  496,  379,
- /*  2170 */   498,    0,  500,    0,    0,    0,    0,  173,    0,    0,
+ /*  2080 */   476,  377,    0,    0,  458,    0,   45,    0,   37,  458,
+ /*  2090 */   240,   37,  420,   37,   37,  240,    0,   37,   37,  240,
+ /*  2100 */    37,    0,    0,  240,    0,   37,    0,   37,  436,    0,
+ /*  2110 */   438,   22,    0,   37,  235,    0,  221,  542,    0,  221,
+ /*  2120 */   484,  222,  215,  487,  213,    0,    0,  491,  492,  493,
+ /*  2130 */   494,  495,  496,    0,  498,  209,  208,    0,    0,  154,
+ /*  2140 */   379,   49,   49,    0,   37,    0,    0,   37,    0,   52,
+ /*  2150 */     0,    0,   49,  392,   49,    0,  484,    0,   45,  487,
+ /*  2160 */     0,    0,    0,  491,  492,  493,  494,  495,  496,  379,
+ /*  2170 */   498,    0,  500,    0,    0,  173,   37,    0,  173,    0,
  /*  2180 */     0,  420,  392,    0,    0,    0,    0,  551,    0,    0,
- /*  2190 */   379,   22,   49,    0,    0,    0,   45,  436,    0,  438,
+ /*  2190 */   379,   49,    0,    0,   45,    0,    0,  436,    0,  438,
  /*  2200 */     0,    0,    0,  392,    0,    0,    0,    0,    0,    0,
  /*  2210 */   420,    0,    0,    0,    0,    0,    0,    0,    0,  379,
- /*  2220 */     0,    0,  461,    0,  154,  153,  436,    0,  438,  152,
- /*  2230 */     0,  420,  392,    0,   66,    0,   66,   50,    0,    0,
- /*  2240 */    50,   22,    0,   66,   22,  484,    0,  436,  487,  438,
- /*  2250 */     0,  461,  491,  492,  493,  494,  495,  496,   66,  498,
- /*  2260 */   420,   37,   37,    0,    0,    0,   37,   14,   42,   37,
- /*  2270 */     0,   37,    0,   42,  484,  204,  436,  487,  438,   43,
- /*  2280 */    52,  491,  492,  493,  494,  495,  496,   42,  498,   52,
- /*  2290 */    33,   42,   52,   45,    0,  484,   49,   42,  487,    0,
- /*  2300 */    49,    0,  491,  492,  493,  494,  495,  496,   49,  498,
- /*  2310 */    42,    0,    0,  379,    0,   49,   49,    0,   74,    0,
- /*  2320 */     0,   37,   42,   52,  484,    0,  392,  487,   37,   52,
- /*  2330 */   379,  491,  492,  493,  494,  495,  496,   42,  498,    0,
- /*  2340 */    37,   52,   42,  392,    0,   37,   42,  379,   52,    0,
- /*  2350 */     0,    0,    0,    0,  420,    0,   22,    0,   37,   22,
- /*  2360 */   392,   37,  121,   37,   37,   37,   37,   37,   37,   37,
- /*  2370 */   436,  420,  438,  119,   33,   37,   33,   37,   22,   37,
- /*  2380 */     0,   22,   37,    0,   22,    0,   22,  436,  420,  438,
- /*  2390 */     0,   22,    0,    0,    0,   37,    0,   37,    0,   22,
- /*  2400 */    54,   20,   37,   37,  436,   37,  438,   37,    0,    0,
- /*  2410 */   379,  114,   49,   37,    0,  113,    0,  220,  484,  113,
- /*  2420 */     0,  487,  226,  392,  225,  491,  492,  493,  494,  495,
- /*  2430 */   496,    3,  498,  190,   22,  484,   33,   22,  487,  301,
- /*  2440 */    33,   37,  491,  492,  493,  494,  495,  496,  190,  498,
- /*  2450 */   190,  420,  484,    0,   37,  487,   50,  196,  111,  491,
- /*  2460 */   492,  493,  494,  495,  496,   33,  498,  436,  190,  438,
- /*  2470 */    33,  190,  216,  200,  200,   50,   49,   33,   49,   80,
- /*  2480 */   113,  109,  301,   33,   37,  114,  113,    3,  114,  114,
- /*  2490 */   379,  113,   33,  114,   37,  113,    0,  113,   37,  113,
- /*  2500 */    37,  114,  113,  392,  301,  114,   37,  379,  113,   37,
- /*  2510 */    37,  114,   33,   49,  114,  484,  114,   49,  487,  114,
- /*  2520 */   392,    0,  491,  492,  493,  494,  495,  496,   42,  498,
- /*  2530 */    42,  420,  113,  379,    0,  114,   42,  114,  114,  113,
- /*  2540 */    33,  193,    2,  113,  113,  111,  392,  436,  420,  438,
- /*  2550 */    22,  197,  113,  111,  285,  272,  249,   49,   49,  114,
- /*  2560 */   114,  113,  113,   22,  436,  113,  438,  113,  113,  113,
- /*  2570 */   379,    0,  114,  113,  420,  252,  114,   42,  113,  113,
- /*  2580 */   113,  193,   49,  392,  114,  114,   22,  113,   22,  113,
- /*  2590 */   436,  113,  438,  192,  113,  484,  113,  122,  487,   22,
- /*  2600 */   123,  114,  491,  492,  493,  494,  495,  496,  113,  498,
- /*  2610 */    37,  420,  484,   37,  114,  487,   37,  114,  113,  491,
- /*  2620 */   492,  493,  494,  495,  496,   37,  498,  436,  193,  438,
- /*  2630 */   114,  379,   37,  114,   37,   37,  226,  114,  484,  113,
- /*  2640 */   134,  487,   33,  113,  392,  491,  492,  493,  494,  495,
- /*  2650 */   496,  134,  498,   37,  379,  113,  134,   22,  134,   74,
- /*  2660 */    73,   22,   37,   37,   37,   37,   37,  392,   37,   37,
- /*  2670 */    37,   37,  420,   37,   37,  484,   80,  107,  487,   80,
+ /*  2220 */     0,   22,  461,  154,    0,  153,  436,    0,  438,  152,
+ /*  2230 */     0,  420,  392,    0,    0,   66,    0,   50,   66,    0,
+ /*  2240 */    50,   22,   66,   22,    0,  484,    0,  436,  487,  438,
+ /*  2250 */     0,  461,  491,  492,  493,  494,  495,  496,    0,  498,
+ /*  2260 */   420,    0,   37,   66,   42,   37,   37,    0,   42,   37,
+ /*  2270 */     0,   14,    0,   42,  484,   52,  436,  487,  438,   37,
+ /*  2280 */    42,  491,  492,  493,  494,  495,  496,   52,  498,   52,
+ /*  2290 */    45,   42,   33,    0,    0,  484,   49,   49,  487,   43,
+ /*  2300 */    49,    0,  491,  492,  493,  494,  495,  496,    0,  498,
+ /*  2310 */    42,    0,  204,  379,   49,    0,    0,    0,   49,    0,
+ /*  2320 */     0,   37,   42,   52,  484,    0,  392,  487,   74,   37,
+ /*  2330 */   379,  491,  492,  493,  494,  495,  496,   52,  498,   42,
+ /*  2340 */     0,   37,   52,  392,   42,    0,   37,  379,   52,   42,
+ /*  2350 */     0,    0,    0,    0,  420,    0,    0,   22,   37,    0,
+ /*  2360 */   392,  121,  119,   22,   37,   37,   37,   37,   37,   37,
+ /*  2370 */   436,  420,  438,    0,   37,   37,   37,   33,    0,   33,
+ /*  2380 */    37,    0,   22,    0,   37,   37,   22,  436,  420,  438,
+ /*  2390 */    22,   22,   22,    0,   37,    0,   54,    0,    0,   37,
+ /*  2400 */    37,    0,   22,   37,  436,   20,  438,    0,   37,   37,
+ /*  2410 */   379,    0,  114,  113,   49,    0,  113,    0,  484,   37,
+ /*  2420 */   226,  487,   22,  392,   22,  491,  492,  493,  494,  495,
+ /*  2430 */   496,  190,  498,    0,    3,  484,   33,  301,  487,  113,
+ /*  2440 */    50,   37,  491,  492,  493,  494,  495,  496,  114,  498,
+ /*  2450 */   190,  420,  484,   37,  190,  487,  190,   33,  196,  491,
+ /*  2460 */   492,  493,  494,  495,  496,  111,  498,  436,  190,  438,
+ /*  2470 */    50,  113,   33,  200,  200,  220,   49,   33,  225,  216,
+ /*  2480 */    33,  114,  301,   49,  109,   80,   33,  113,   37,  113,
+ /*  2490 */   379,  114,    3,  114,   33,   37,   37,  113,   49,  113,
+ /*  2500 */    37,  114,  113,  392,  301,  114,  114,  379,  114,  113,
+ /*  2510 */    37,   37,   37,  114,   33,  484,   49,    0,  487,    0,
+ /*  2520 */   392,  113,  491,  492,  493,  494,  495,  496,  114,  498,
+ /*  2530 */    42,  420,  114,  379,    0,  114,  113,   42,  113,  113,
+ /*  2540 */     0,  113,  193,  114,   42,   33,  392,  436,  420,  438,
+ /*  2550 */     2,  111,  111,  272,   22,  249,  114,   49,   49,  114,
+ /*  2560 */   114,  285,  114,  113,  436,  113,  438,  113,  113,  113,
+ /*  2570 */   379,   22,    0,  113,  420,  113,  252,  113,  113,   42,
+ /*  2580 */   114,  113,  113,  392,   49,  197,  114,   37,   22,  113,
+ /*  2590 */   436,  113,  438,  122,  114,  484,  113,  113,  487,  193,
+ /*  2600 */   123,  192,  491,  492,  493,  494,  495,  496,   22,  498,
+ /*  2610 */    37,  420,  484,   22,  113,  487,   37,  114,  113,  491,
+ /*  2620 */   492,  493,  494,  495,  496,  193,  498,  436,  114,  438,
+ /*  2630 */    37,  379,  114,   37,  114,   37,  226,  114,  484,   37,
+ /*  2640 */   134,  487,  113,   33,  392,  491,  492,  493,  494,  495,
+ /*  2650 */   496,  113,  498,   37,  379,  134,  134,  113,  134,   22,
+ /*  2660 */    22,   74,   73,   37,   37,   37,   37,  392,   37,   37,
+ /*  2670 */    37,   37,  420,   37,   80,  484,   37,   37,  487,   80,
  /*  2680 */    33,  107,  491,  492,  493,  494,  495,  496,  436,  498,
- /*  2690 */   438,   37,   37,   37,   22,  420,   37,   37,   37,   80,
- /*  2700 */    37,   37,   37,   37,   37,   22,   37,    0,   37,   52,
- /*  2710 */    42,  436,    0,  438,   37,  379,   42,    0,   37,    0,
- /*  2720 */    42,   37,   52,    0,   42,   52,   37,   52,  392,   37,
- /*  2730 */    22,    0,   22,   33,  554,   22,  484,    0,   22,  487,
- /*  2740 */   379,   21,   21,  491,  492,  493,  494,  495,  496,   22,
- /*  2750 */   498,  554,   20,  392,  554,  554,  420,  554,  554,  484,
+ /*  2690 */   438,   37,  107,   37,   37,  420,   22,   37,   37,   37,
+ /*  2700 */    80,   37,   37,   37,   37,   37,   22,   37,    0,   37,
+ /*  2710 */    52,  436,    0,  438,   37,  379,    0,   42,   42,   37,
+ /*  2720 */    42,   52,    0,   52,   37,   52,    0,   42,  392,   37,
+ /*  2730 */     0,   22,   37,    0,   33,   22,  484,   22,   21,  487,
+ /*  2740 */   379,   22,   22,  491,  492,  493,  494,  495,  496,   21,
+ /*  2750 */   498,   20,  554,  392,  554,  554,  420,  554,  554,  484,
  /*  2760 */   554,  554,  487,  554,  554,  554,  491,  492,  493,  494,
  /*  2770 */   495,  496,  436,  498,  438,  554,  554,  554,  554,  554,
  /*  2780 */   554,  420,  554,  554,  554,  554,  554,  554,  554,  554,
@@ -1284,9 +1287,9 @@ static const YYCODETYPE yy_lookahead[] = {
  /*  3590 */   376,  376,  376,  376,  376,  376,  376,  376,  376,  376,
  /*  3600 */   376,  376,  376,  376,
 };
-#define YY_SHIFT_COUNT    (962)
+#define YY_SHIFT_COUNT    (964)
 #define YY_SHIFT_MIN      (0)
-#define YY_SHIFT_MAX      (2737)
+#define YY_SHIFT_MAX      (2733)
 static const unsigned short int yy_shift_ofst[] = {
  /*     0 */   520,    0,  259,    0,  519,  519,  519,  519,  519,  519,
  /*    10 */   519,  519,  519,  519,  519,  519,  778, 1037, 1037, 1296,
@@ -1326,67 +1329,67 @@ static const unsigned short int yy_shift_ofst[] = {
  /*   350 */   503,  503,  503,  503,  503,  503,  503,  503,  503,  503,
  /*   360 */   503,  503,  503,  503,  503,  503,  503,  503, 1904,  503,
  /*   370 */  1701,  404, 1996, 1983, 1774, 3228, 3228, 3228, 3228, 3228,
- /*   380 */  3228, 3228, 3228, 3228, 3228, 3228,   81, 1425,  359, 1684,
- /*   390 */   391,   17,  558,   63,  707, 1492,  231,  724,  588,  588,
- /*   400 */   588,  588,  588,  588,  588,  588,  588,  750,  610,  464,
- /*   410 */    66,  723,  723,  402,  733,   39,  782,  578,  760,  846,
- /*   420 */   578,  889,  802, 1075,   54,  302,  302, 1231, 1113,  964,
- /*   430 */  1231, 1231, 1231, 1404, 1255, 1336,  605, 1151,  696, 1440,
- /*   440 */  1313, 1315, 1317, 1328,  734, 1441, 1442, 1476, 1480, 1490,
- /*   450 */  1493, 1226, 1416, 1420,    8, 1477, 1481, 1485, 1487, 1332,
- /*   460 */  1303,  621, 1483, 1496, 1499, 1541, 1502,  781, 1511, 1409,
- /*   470 */  1535, 1538, 1547, 1550, 1526, 1560, 1572, 1577, 1592, 1602,
- /*   480 */  1630, 1635, 1642, 1644, 1647, 1563, 1565, 1567, 1573, 1591,
- /*   490 */  1600,  429, 1446,  498, 1589, 1504, 1632, 1621, 1390, 2080,
- /*   500 */  2082, 2083, 2040, 2087, 2051, 1849, 2053, 2054, 2056, 1855,
- /*   510 */  2094, 2059, 2060, 1858, 2062, 2100, 2101, 1862, 2103, 2067,
- /*   520 */  2105, 2069, 2107, 2089, 2109, 2075, 1878, 2114, 1894, 2116,
- /*   530 */  1897, 1899, 1907, 1906, 2124, 2125, 2126, 1924, 1927, 2136,
- /*   540 */  2137, 1984, 2090, 2092, 2142, 2106, 2144, 2145, 2110, 2096,
- /*   550 */  2146, 2102, 2149, 2112, 2150, 2152, 2158, 2113, 2160, 2161,
- /*   560 */  2171, 2173, 2174, 2175, 1981, 2118, 2176, 2004, 2178, 2179,
- /*   570 */  2180, 2183, 2184, 2185, 2186, 2188, 2189, 2198, 2200, 2201,
- /*   580 */  2202, 2204, 2205, 2206, 2207, 2208, 2209, 2211, 2143, 2193,
- /*   590 */  2151, 2194, 2195, 2212, 2213, 2214, 2215, 2216, 2217, 2218,
- /*   600 */  2169, 2220, 2070, 2221, 2072, 2223, 2077, 2227, 2230, 2219,
- /*   610 */  2187, 2222, 2190, 2233, 2168, 2235, 2170, 2224, 2238, 2177,
- /*   620 */  2239, 2192, 2242, 2246, 2225, 2228, 2226, 2250, 2229, 2237,
- /*   630 */  2231, 2263, 2232, 2240, 2245, 2264, 2234, 2265, 2248, 2249,
- /*   640 */  2257, 2247, 2251, 2253, 2259, 2270, 2236, 2255, 2272, 2294,
- /*   650 */  2299, 2301, 2268, 2071, 2311, 2247, 2266, 2312, 2247, 2267,
- /*   660 */  2314, 2317, 2244, 2319, 2320, 2284, 2271, 2280, 2325, 2291,
- /*   670 */  2277, 2295, 2339, 2303, 2289, 2300, 2344, 2308, 2296, 2304,
- /*   680 */  2349, 2350, 2351, 2352, 2353, 2355, 2241, 2254, 2321, 2334,
- /*   690 */  2357, 2337, 2324, 2326, 2327, 2328, 2329, 2330, 2331, 2332,
- /*   700 */  2338, 2341, 2343, 2340, 2342, 2356, 2345, 2380, 2359, 2383,
- /*   710 */  2362, 2385, 2364, 2346, 2390, 2369, 2358, 2392, 2393, 2394,
- /*   720 */  2360, 2396, 2365, 2398, 2377, 2381, 2366, 2368, 2370, 2297,
- /*   730 */  2302, 2408, 2243, 2196, 2199, 2306, 2197, 2247, 2363, 2409,
- /*   740 */  2258, 2376, 2412, 2414, 2256, 2415, 2260, 2261, 2416, 2420,
- /*   750 */  2278, 2273, 2281, 2274, 2428, 2403, 2138, 2367, 2371, 2373,
- /*   760 */  2374, 2404, 2417, 2378, 2406, 2347, 2425, 2372, 2375, 2407,
- /*   770 */  2432, 2379, 2382, 2384, 2386, 2387, 2437, 2427, 2429, 2389,
- /*   780 */  2444, 2181, 2399, 2391, 2450, 2395, 2447, 2397, 2400, 2484,
- /*   790 */  2459, 2203, 2457, 2461, 2463, 2469, 2472, 2473, 2402, 2405,
- /*   800 */  2464, 2269, 2479, 2468, 2453, 2496, 2419, 2486, 2421, 2423,
- /*   810 */  2426, 2430, 2348, 2431, 2521, 2488, 2354, 2534, 2424, 2439,
- /*   820 */  2388, 2494, 2401, 2507, 2434, 2283, 2442, 2540, 2528, 2307,
- /*   830 */  2445, 2446, 2448, 2449, 2452, 2454, 2455, 2458, 2508, 2456,
- /*   840 */  2460, 2509, 2462, 2541, 2323, 2465, 2466, 2571, 2470, 2467,
- /*   850 */  2435, 2535, 2474, 2475, 2247, 2533, 2476, 2478, 2471, 2481,
- /*   860 */  2483, 2477, 2564, 2566, 2577, 2410, 2487, 2573, 2576, 2495,
- /*   870 */  2500, 2579, 2505, 2503, 2588, 2448, 2516, 2595, 2449, 2519,
- /*   880 */  2597, 2452, 2523, 2598, 2454, 2506, 2517, 2522, 2524, 2526,
- /*   890 */  2609, 2530, 2616, 2542, 2609, 2609, 2635, 2585, 2587, 2639,
- /*   900 */  2625, 2626, 2627, 2628, 2629, 2631, 2632, 2633, 2634, 2636,
- /*   910 */  2637, 2596, 2570, 2599, 2574, 2647, 2654, 2655, 2656, 2672,
- /*   920 */  2659, 2660, 2661, 2619, 2341, 2663, 2343, 2664, 2665, 2666,
- /*   930 */  2667, 2683, 2669, 2707, 2671, 2657, 2668, 2712, 2677, 2670,
- /*   940 */  2674, 2717, 2681, 2673, 2678, 2719, 2684, 2675, 2682, 2723,
- /*   950 */  2689, 2737, 2708, 2692, 2731, 2710, 2700, 2713, 2720, 2716,
- /*   960 */  2727, 2721, 2732,
+ /*   380 */  3228, 3228, 3228, 3228, 3228, 3228, 3228,   81, 1425,  359,
+ /*   390 */  1684,  391,   17,  558,   63,  707, 1492,  231,  724,  588,
+ /*   400 */   588,  588,  588,  588,  588,  588,  588,  588,  750,  610,
+ /*   410 */   464,   66,  723,  723,  402,  733,   39,  782,  578,  578,
+ /*   420 */   760,  846,  578,  889,  802, 1075,   54,  302,  302, 1231,
+ /*   430 */  1113,  964, 1231, 1231, 1231, 1404, 1255, 1336,  605, 1151,
+ /*   440 */   696, 1440, 1313, 1315, 1317, 1328,  734, 1441, 1442, 1476,
+ /*   450 */  1480, 1490, 1493, 1226, 1416, 1420,    8, 1477, 1481, 1485,
+ /*   460 */  1487, 1332, 1303,  621, 1483, 1496, 1499, 1541, 1502,  781,
+ /*   470 */  1511, 1409, 1535, 1538, 1547, 1550, 1526, 1560, 1572, 1577,
+ /*   480 */  1592, 1602, 1630, 1635, 1642, 1644, 1647, 1563, 1565, 1567,
+ /*   490 */  1573, 1591, 1600,  429, 1446,  498, 1589, 1504, 1632, 1621,
+ /*   500 */  1390, 2082, 2083, 2085, 2041, 2087, 2051, 1850, 2054, 2056,
+ /*   510 */  2057, 1855, 2096, 2060, 2061, 1859, 2063, 2101, 2102, 1863,
+ /*   520 */  2104, 2068, 2106, 2070, 2109, 2089, 2112, 2076, 1879, 2115,
+ /*   530 */  1895, 2118, 1898, 1899, 1907, 1911, 2125, 2126, 2133, 1926,
+ /*   540 */  1928, 2137, 2138, 1985, 2092, 2093, 2143, 2107, 2145, 2146,
+ /*   550 */  2110, 2097, 2148, 2103, 2150, 2113, 2151, 2157, 2160, 2105,
+ /*   560 */  2155, 2161, 2162, 2171, 2173, 2174, 2002, 2139, 2177, 2005,
+ /*   570 */  2179, 2180, 2183, 2184, 2185, 2186, 2188, 2189, 2198, 2200,
+ /*   580 */  2201, 2202, 2204, 2205, 2206, 2207, 2208, 2209, 2211, 2212,
+ /*   590 */  2142, 2192, 2149, 2193, 2195, 2196, 2213, 2214, 2215, 2216,
+ /*   600 */  2217, 2218, 2199, 2220, 2069, 2224, 2072, 2227, 2077, 2230,
+ /*   610 */  2233, 2219, 2187, 2221, 2190, 2234, 2169, 2236, 2172, 2225,
+ /*   620 */  2239, 2176, 2244, 2197, 2246, 2250, 2228, 2223, 2222, 2258,
+ /*   630 */  2229, 2235, 2226, 2261, 2232, 2237, 2231, 2267, 2242, 2270,
+ /*   640 */  2245, 2238, 2259, 2247, 2248, 2257, 2251, 2272, 2256, 2249,
+ /*   650 */  2293, 2294, 2301, 2308, 2268, 2108, 2311, 2247, 2265, 2315,
+ /*   660 */  2247, 2269, 2316, 2317, 2254, 2319, 2320, 2284, 2271, 2280,
+ /*   670 */  2325, 2292, 2285, 2297, 2340, 2304, 2290, 2302, 2345, 2309,
+ /*   680 */  2296, 2307, 2350, 2351, 2352, 2353, 2355, 2356, 2240, 2243,
+ /*   690 */  2321, 2335, 2359, 2341, 2327, 2328, 2329, 2330, 2331, 2332,
+ /*   700 */  2337, 2338, 2339, 2344, 2346, 2343, 2347, 2360, 2348, 2373,
+ /*   710 */  2364, 2378, 2368, 2381, 2369, 2342, 2383, 2370, 2357, 2393,
+ /*   720 */  2395, 2397, 2362, 2398, 2363, 2401, 2380, 2385, 2366, 2371,
+ /*   730 */  2372, 2298, 2300, 2407, 2241, 2194, 2253, 2303, 2255, 2247,
+ /*   740 */  2365, 2411, 2260, 2382, 2400, 2415, 2263, 2402, 2264, 2262,
+ /*   750 */  2417, 2433, 2266, 2273, 2278, 2274, 2431, 2403, 2136, 2326,
+ /*   760 */  2334, 2358, 2367, 2404, 2416, 2374, 2390, 2354, 2420, 2375,
+ /*   770 */  2377, 2424, 2439, 2379, 2376, 2384, 2386, 2387, 2444, 2427,
+ /*   780 */  2434, 2389, 2447, 2181, 2405, 2391, 2453, 2396, 2451, 2392,
+ /*   790 */  2394, 2489, 2461, 2203, 2458, 2459, 2463, 2473, 2474, 2475,
+ /*   800 */  2399, 2414, 2449, 2276, 2481, 2467, 2517, 2519, 2408, 2488,
+ /*   810 */  2418, 2421, 2423, 2425, 2349, 2426, 2534, 2495, 2388, 2540,
+ /*   820 */  2429, 2428, 2406, 2502, 2409, 2512, 2440, 2281, 2441, 2548,
+ /*   830 */  2532, 2306, 2442, 2445, 2450, 2452, 2454, 2455, 2456, 2446,
+ /*   840 */  2508, 2460, 2462, 2509, 2448, 2549, 2324, 2464, 2465, 2572,
+ /*   850 */  2466, 2468, 2432, 2537, 2469, 2471, 2247, 2535, 2476, 2478,
+ /*   860 */  2472, 2483, 2484, 2477, 2566, 2586, 2591, 2410, 2480, 2550,
+ /*   870 */  2573, 2501, 2503, 2579, 2505, 2514, 2593, 2450, 2518, 2596,
+ /*   880 */  2452, 2520, 2598, 2454, 2523, 2602, 2455, 2506, 2521, 2522,
+ /*   890 */  2524, 2529, 2610, 2538, 2616, 2544, 2610, 2610, 2637, 2587,
+ /*   900 */  2589, 2638, 2626, 2627, 2628, 2629, 2631, 2632, 2633, 2634,
+ /*   910 */  2636, 2639, 2640, 2594, 2574, 2599, 2585, 2647, 2654, 2656,
+ /*   920 */  2657, 2674, 2660, 2661, 2662, 2620, 2344, 2664, 2346, 2665,
+ /*   930 */  2666, 2667, 2668, 2684, 2670, 2708, 2672, 2658, 2675, 2712,
+ /*   940 */  2677, 2669, 2676, 2716, 2682, 2671, 2678, 2722, 2687, 2673,
+ /*   950 */  2685, 2726, 2692, 2730, 2709, 2695, 2733, 2713, 2701, 2715,
+ /*   960 */  2717, 2719, 2720, 2728, 2731,
 };
-#define YY_REDUCE_COUNT (385)
+#define YY_REDUCE_COUNT (386)
 #define YY_REDUCE_MIN   (-510)
 #define YY_REDUCE_MAX   (2729)
 static const short yy_reduce_ofst[] = {
@@ -1427,107 +1430,107 @@ static const short yy_reduce_ofst[] = {
  /*   340 */  1555, 1559, 1610, 1594, 1614, 1622, 1580, 1583, 1584, 1587,
  /*   350 */  1588, 1590, 1593, 1595, 1596, 1598, 1599, 1607, 1609, 1612,
  /*   360 */  1613, 1617, 1619, 1624, 1625, 1627, 1629, 1640, 1597, 1641,
- /*   370 */  1634, 1638, 1646, 1662, 1668, 1579, 1571, 1574, 1626, 1628,
- /*   380 */  1643, 1664, 1653, 1665, 1659, 1704,
+ /*   370 */  1634, 1638, 1646, 1662, 1668, 1579, 1628, 1571, 1604, 1626,
+ /*   380 */  1631, 1643, 1664, 1653, 1665, 1659, 1704,
 };
 static const YYACTIONTYPE yy_default[] = {
- /*     0 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    10 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    20 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    30 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    40 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    50 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    60 */  2547, 2179, 2179, 2503, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    70 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2510, 2179,
- /*    80 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*    90 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2280,
- /*   100 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   110 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2278, 2799,
- /*   120 */  2179, 2925, 2588, 2179, 2179, 2828, 2179, 2179, 2179, 2179,
- /*   130 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   140 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2811, 2179, 2179,
- /*   150 */  2251, 2251, 2179, 2811, 2811, 2811, 2771, 2771, 2278, 2179,
- /*   160 */  2179, 2280, 2179, 2590, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   170 */  2179, 2179, 2179, 2420, 2209, 2179, 2179, 2179, 2179, 2179,
- /*   180 */  2179, 2573, 2179, 2179, 2857, 2803, 2804, 2919, 2179, 2860,
- /*   190 */  2822, 2179, 2817, 2179, 2179, 2179, 2179, 2515, 2179, 2847,
- /*   200 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2616, 2179, 2179,
- /*   210 */  2365, 2567, 2179, 2179, 2179, 2179, 2179, 2903, 2801, 2841,
- /*   220 */  2179, 2851, 2179, 2179, 2179, 2604, 2280, 2179, 2280, 2560,
- /*   230 */  2498, 2179, 2508, 2179, 2508, 2505, 2179, 2179, 2179, 2508,
- /*   240 */  2505, 2505, 2505, 2354, 2350, 2179, 2179, 2348, 2179, 2179,
- /*   250 */  2179, 2179, 2234, 2179, 2234, 2179, 2280, 2280, 2179, 2280,
- /*   260 */  2179, 2179, 2280, 2179, 2280, 2179, 2280, 2179, 2280, 2280,
- /*   270 */  2179, 2280, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   280 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   290 */  2179, 2602, 2583, 2179, 2278, 2179, 2571, 2569, 2179, 2278,
- /*   300 */  2851, 2179, 2179, 2873, 2868, 2873, 2868, 2887, 2883, 2873,
- /*   310 */  2892, 2889, 2853, 2851, 2834, 2830, 2922, 2909, 2905, 2179,
- /*   320 */  2179, 2839, 2837, 2179, 2278, 2278, 2868, 2179, 2179, 2179,
- /*   330 */  2179, 2868, 2179, 2179, 2278, 2179, 2179, 2278, 2179, 2179,
- /*   340 */  2179, 2179, 2278, 2179, 2278, 2179, 2179, 2179, 2179, 2179,
- /*   350 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   360 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2382, 2179,
- /*   370 */  2179, 2278, 2179, 2218, 2179, 2562, 2588, 2593, 2543, 2543,
- /*   380 */  2423, 2423, 2925, 2423, 2281, 2184, 2179, 2179, 2179, 2179,
- /*   390 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2886, 2885,
- /*   400 */  2722, 2179, 2775, 2774, 2773, 2764, 2721, 2378, 2179, 2179,
- /*   410 */  2179, 2720, 2719, 2179, 2179, 2179, 2179, 2369, 2179, 2179,
- /*   420 */  2391, 2179, 2179, 2179, 2179, 2534, 2533, 2713, 2179, 2179,
- /*   430 */  2714, 2712, 2711, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   440 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   450 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   460 */  2906, 2910, 2179, 2179, 2179, 2800, 2179, 2179, 2179, 2692,
- /*   470 */  2179, 2179, 2179, 2179, 2660, 2655, 2646, 2637, 2652, 2643,
- /*   480 */  2631, 2649, 2640, 2628, 2625, 2179, 2179, 2179, 2179, 2179,
- /*   490 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   500 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   510 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   520 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   530 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   540 */  2179, 2504, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   550 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   560 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   570 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   580 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   590 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   600 */  2179, 2179, 2179, 2179, 2179, 2179, 2519, 2179, 2179, 2179,
- /*   610 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   620 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   630 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   640 */  2223, 2699, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   650 */  2179, 2179, 2179, 2179, 2179, 2702, 2179, 2179, 2703, 2179,
- /*   660 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   670 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   680 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   690 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   700 */  2179, 2325, 2324, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   710 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   720 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2704,
- /*   730 */  2179, 2179, 2179, 2179, 2587, 2179, 2179, 2694, 2179, 2179,
- /*   740 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   750 */  2179, 2179, 2179, 2179, 2902, 2854, 2179, 2179, 2179, 2179,
- /*   760 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   770 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2692, 2179,
- /*   780 */  2884, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2900, 2179,
- /*   790 */  2904, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2810, 2806,
- /*   800 */  2179, 2179, 2802, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   810 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   820 */  2179, 2179, 2179, 2761, 2179, 2179, 2179, 2795, 2179, 2179,
- /*   830 */  2179, 2179, 2419, 2418, 2417, 2416, 2179, 2179, 2179, 2179,
- /*   840 */  2179, 2179, 2704, 2179, 2707, 2179, 2179, 2179, 2179, 2179,
- /*   850 */  2179, 2179, 2179, 2179, 2691, 2179, 2746, 2745, 2179, 2179,
- /*   860 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2413,
- /*   870 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   880 */  2179, 2179, 2179, 2179, 2179, 2397, 2395, 2394, 2393, 2179,
- /*   890 */  2430, 2179, 2179, 2179, 2426, 2425, 2179, 2179, 2179, 2179,
- /*   900 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   910 */  2179, 2179, 2179, 2179, 2179, 2299, 2179, 2179, 2179, 2179,
- /*   920 */  2179, 2179, 2179, 2179, 2291, 2179, 2290, 2179, 2179, 2179,
- /*   930 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   940 */  2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179,
- /*   950 */  2179, 2179, 2179, 2179, 2179, 2179, 2208, 2179, 2179, 2179,
- /*   960 */  2179, 2179, 2179,
+ /*     0 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    10 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    20 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    30 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    40 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    50 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    60 */  2548, 2180, 2180, 2504, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    70 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2511, 2180,
+ /*    80 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*    90 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2281,
+ /*   100 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   110 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2279, 2800,
+ /*   120 */  2180, 2926, 2589, 2180, 2180, 2829, 2180, 2180, 2180, 2180,
+ /*   130 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   140 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2812, 2180, 2180,
+ /*   150 */  2252, 2252, 2180, 2812, 2812, 2812, 2772, 2772, 2279, 2180,
+ /*   160 */  2180, 2281, 2180, 2591, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   170 */  2180, 2180, 2180, 2421, 2210, 2180, 2180, 2180, 2180, 2180,
+ /*   180 */  2180, 2574, 2180, 2180, 2858, 2804, 2805, 2920, 2180, 2861,
+ /*   190 */  2823, 2180, 2818, 2180, 2180, 2180, 2180, 2516, 2180, 2848,
+ /*   200 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2617, 2180, 2180,
+ /*   210 */  2366, 2568, 2180, 2180, 2180, 2180, 2180, 2904, 2802, 2842,
+ /*   220 */  2180, 2852, 2180, 2180, 2180, 2605, 2281, 2180, 2281, 2561,
+ /*   230 */  2499, 2180, 2509, 2180, 2509, 2506, 2180, 2180, 2180, 2509,
+ /*   240 */  2506, 2506, 2506, 2355, 2351, 2180, 2180, 2349, 2180, 2180,
+ /*   250 */  2180, 2180, 2235, 2180, 2235, 2180, 2281, 2281, 2180, 2281,
+ /*   260 */  2180, 2180, 2281, 2180, 2281, 2180, 2281, 2180, 2281, 2281,
+ /*   270 */  2180, 2281, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   280 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   290 */  2180, 2603, 2584, 2180, 2279, 2180, 2572, 2570, 2180, 2279,
+ /*   300 */  2852, 2180, 2180, 2874, 2869, 2874, 2869, 2888, 2884, 2874,
+ /*   310 */  2893, 2890, 2854, 2852, 2835, 2831, 2923, 2910, 2906, 2180,
+ /*   320 */  2180, 2840, 2838, 2180, 2279, 2279, 2869, 2180, 2180, 2180,
+ /*   330 */  2180, 2869, 2180, 2180, 2279, 2180, 2180, 2279, 2180, 2180,
+ /*   340 */  2180, 2180, 2279, 2180, 2279, 2180, 2180, 2180, 2180, 2180,
+ /*   350 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   360 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2383, 2180,
+ /*   370 */  2180, 2279, 2180, 2219, 2180, 2563, 2926, 2589, 2594, 2544,
+ /*   380 */  2544, 2424, 2424, 2926, 2424, 2282, 2185, 2180, 2180, 2180,
+ /*   390 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2887,
+ /*   400 */  2886, 2723, 2180, 2776, 2775, 2774, 2765, 2722, 2379, 2180,
+ /*   410 */  2180, 2180, 2721, 2720, 2180, 2180, 2180, 2180, 2370, 2367,
+ /*   420 */  2180, 2180, 2392, 2180, 2180, 2180, 2180, 2535, 2534, 2714,
+ /*   430 */  2180, 2180, 2715, 2713, 2712, 2180, 2180, 2180, 2180, 2180,
+ /*   440 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   450 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   460 */  2180, 2180, 2907, 2911, 2180, 2180, 2180, 2801, 2180, 2180,
+ /*   470 */  2180, 2693, 2180, 2180, 2180, 2180, 2661, 2656, 2647, 2638,
+ /*   480 */  2653, 2644, 2632, 2650, 2641, 2629, 2626, 2180, 2180, 2180,
+ /*   490 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   500 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   510 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   520 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   530 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   540 */  2180, 2180, 2180, 2505, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   550 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   560 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   570 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   580 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   590 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   600 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2520, 2180,
+ /*   610 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   620 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   630 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   640 */  2180, 2180, 2224, 2700, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   650 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2703, 2180, 2180,
+ /*   660 */  2704, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   670 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   680 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   690 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   700 */  2180, 2180, 2180, 2326, 2325, 2180, 2180, 2180, 2180, 2180,
+ /*   710 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   720 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   730 */  2180, 2705, 2180, 2180, 2180, 2180, 2588, 2180, 2180, 2695,
+ /*   740 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   750 */  2180, 2180, 2180, 2180, 2180, 2180, 2903, 2855, 2180, 2180,
+ /*   760 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   770 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   780 */  2693, 2180, 2885, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   790 */  2901, 2180, 2905, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   800 */  2811, 2807, 2180, 2180, 2803, 2180, 2180, 2180, 2180, 2180,
+ /*   810 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   820 */  2180, 2180, 2180, 2180, 2180, 2762, 2180, 2180, 2180, 2796,
+ /*   830 */  2180, 2180, 2180, 2180, 2420, 2419, 2418, 2417, 2180, 2180,
+ /*   840 */  2180, 2180, 2180, 2180, 2705, 2180, 2708, 2180, 2180, 2180,
+ /*   850 */  2180, 2180, 2180, 2180, 2180, 2180, 2692, 2180, 2747, 2746,
+ /*   860 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   870 */  2180, 2414, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   880 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2398, 2396, 2395,
+ /*   890 */  2394, 2180, 2431, 2180, 2180, 2180, 2427, 2426, 2180, 2180,
+ /*   900 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   910 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2300, 2180, 2180,
+ /*   920 */  2180, 2180, 2180, 2180, 2180, 2180, 2292, 2180, 2291, 2180,
+ /*   930 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   940 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180,
+ /*   950 */  2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2209, 2180,
+ /*   960 */  2180, 2180, 2180, 2180, 2180,
 };
 /********** End of lemon-generated parsing tables *****************************/
 
@@ -2756,7 +2759,7 @@ static const char *const yyRuleName[] = {
  /* 181 */ "cmd ::= ALTER TABLE alter_table_clause",
  /* 182 */ "cmd ::= ALTER STABLE alter_table_clause",
  /* 183 */ "alter_table_clause ::= full_table_name alter_table_options",
- /* 184 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name",
+ /* 184 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options",
  /* 185 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name",
  /* 186 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name",
  /* 187 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options",
@@ -3533,7 +3536,9 @@ static void yy_destructor(
     case 549: /* query_simple_or_subquery */
     case 551: /* sort_specification */
 {
+#line 7 "sql.y"
  nodesDestroyNode((yypminor->yy600)); 
+#line 3541 "sql.c"
 }
       break;
     case 377: /* account_options */
@@ -3543,7 +3548,9 @@ static void yy_destructor(
     case 462: /* with_meta */
     case 471: /* bufsize_opt */
 {
+#line 54 "sql.y"
  
+#line 3553 "sql.c"
 }
       break;
     case 381: /* ip_range_list */
@@ -3583,7 +3590,9 @@ static void yy_destructor(
     case 545: /* order_by_clause_opt */
     case 550: /* sort_specification_list */
 {
+#line 85 "sql.y"
  nodesDestroyList((yypminor->yy748)); 
+#line 3595 "sql.c"
 }
       break;
     case 384: /* user_name */
@@ -3607,24 +3616,32 @@ static void yy_destructor(
     case 498: /* noarg_func */
     case 516: /* alias_opt */
 {
+#line 1069 "sql.y"
  
+#line 3621 "sql.c"
 }
       break;
     case 385: /* sysinfo_opt */
 {
+#line 112 "sql.y"
  
+#line 3628 "sql.c"
 }
       break;
     case 386: /* privileges */
     case 389: /* priv_type_list */
     case 390: /* priv_type */
 {
+#line 121 "sql.y"
  
+#line 3637 "sql.c"
 }
       break;
     case 387: /* priv_level */
 {
+#line 138 "sql.y"
  
+#line 3644 "sql.c"
 }
       break;
     case 396: /* force_opt */
@@ -3638,66 +3655,90 @@ static void yy_destructor(
     case 527: /* set_quantifier_opt */
     case 528: /* tag_mode_opt */
 {
+#line 170 "sql.y"
  
+#line 3660 "sql.c"
 }
       break;
     case 409: /* alter_db_option */
     case 435: /* alter_table_option */
 {
+#line 278 "sql.y"
  
+#line 3668 "sql.c"
 }
       break;
     case 421: /* type_name */
     case 432: /* type_name_default_len */
 {
+#line 413 "sql.y"
  
+#line 3676 "sql.c"
 }
       break;
     case 440: /* db_kind_opt */
     case 447: /* table_kind */
 {
+#line 591 "sql.y"
  
+#line 3684 "sql.c"
 }
       break;
     case 441: /* table_kind_db_name_cond_opt */
 {
+#line 556 "sql.y"
  
+#line 3691 "sql.c"
 }
       break;
     case 451: /* tsma_func_list */
 {
+#line 610 "sql.y"
  nodesDestroyNode((yypminor->yy600)); 
+#line 3698 "sql.c"
 }
       break;
     case 506: /* compare_op */
     case 507: /* in_op */
 {
+#line 1267 "sql.y"
  
+#line 3706 "sql.c"
 }
       break;
     case 519: /* join_type */
 {
+#line 1348 "sql.y"
  
+#line 3713 "sql.c"
 }
       break;
     case 520: /* join_subtype */
 {
+#line 1356 "sql.y"
  
+#line 3720 "sql.c"
 }
       break;
     case 541: /* fill_mode */
 {
+#line 1472 "sql.y"
  
+#line 3727 "sql.c"
 }
       break;
     case 552: /* ordering_specification_opt */
 {
+#line 1557 "sql.y"
  
+#line 3734 "sql.c"
 }
       break;
     case 553: /* null_ordering_opt */
 {
+#line 1563 "sql.y"
  
+#line 3741 "sql.c"
 }
       break;
 /********* End destructor definitions *****************************************/
@@ -4170,7 +4211,7 @@ static const YYCODETYPE yyRuleInfoLhs[] = {
    376,  /* (181) cmd ::= ALTER TABLE alter_table_clause */
    376,  /* (182) cmd ::= ALTER STABLE alter_table_clause */
    418,  /* (183) alter_table_clause ::= full_table_name alter_table_options */
-   418,  /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */
+   418,  /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options */
    418,  /* (185) alter_table_clause ::= full_table_name DROP COLUMN column_name */
    418,  /* (186) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */
    418,  /* (187) alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options */
@@ -4923,7 +4964,7 @@ static const signed char yyRuleInfoNRhs[] = {
    -3,  /* (181) cmd ::= ALTER TABLE alter_table_clause */
    -3,  /* (182) cmd ::= ALTER STABLE alter_table_clause */
    -2,  /* (183) alter_table_clause ::= full_table_name alter_table_options */
-   -5,  /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */
+   -6,  /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options */
    -4,  /* (185) alter_table_clause ::= full_table_name DROP COLUMN column_name */
    -5,  /* (186) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */
    -5,  /* (187) alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options */
@@ -5529,15 +5570,21 @@ static YYACTIONTYPE yy_reduce(
 /********** Begin reduce actions **********************************************/
         YYMINORTYPE yylhsminor;
       case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */
+#line 50 "sql.y"
 { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
+#line 5575 "sql.c"
   yy_destructor(yypParser,377,&yymsp[0].minor);
         break;
       case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */
+#line 51 "sql.y"
 { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
+#line 5581 "sql.c"
   yy_destructor(yypParser,378,&yymsp[0].minor);
         break;
       case 2: /* account_options ::= */
+#line 55 "sql.y"
 { }
+#line 5587 "sql.c"
         break;
       case 3: /* account_options ::= account_options PPS literal */
       case 4: /* account_options ::= account_options TSERIES literal */ yytestcase(yyruleno==4);
@@ -5549,18 +5596,24 @@ static YYACTIONTYPE yy_reduce(
       case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10);
       case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11);
 {  yy_destructor(yypParser,377,&yymsp[-2].minor);
+#line 56 "sql.y"
 { }
+#line 5601 "sql.c"
   yy_destructor(yypParser,379,&yymsp[0].minor);
 }
         break;
       case 12: /* alter_account_options ::= alter_account_option */
 {  yy_destructor(yypParser,380,&yymsp[0].minor);
+#line 68 "sql.y"
 { }
+#line 5609 "sql.c"
 }
         break;
       case 13: /* alter_account_options ::= alter_account_options alter_account_option */
 {  yy_destructor(yypParser,378,&yymsp[-1].minor);
+#line 69 "sql.y"
 { }
+#line 5616 "sql.c"
   yy_destructor(yypParser,380,&yymsp[0].minor);
 }
         break;
@@ -5574,19 +5627,27 @@ static YYACTIONTYPE yy_reduce(
       case 21: /* alter_account_option ::= USERS literal */ yytestcase(yyruleno==21);
       case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22);
       case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23);
+#line 73 "sql.y"
 { }
+#line 5632 "sql.c"
   yy_destructor(yypParser,379,&yymsp[0].minor);
         break;
       case 24: /* ip_range_list ::= NK_STRING */
+#line 86 "sql.y"
 { yylhsminor.yy748 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); }
+#line 5638 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 25: /* ip_range_list ::= ip_range_list NK_COMMA NK_STRING */
+#line 87 "sql.y"
 { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); }
+#line 5644 "sql.c"
   yymsp[-2].minor.yy748 = yylhsminor.yy748;
         break;
       case 26: /* white_list ::= HOST ip_range_list */
+#line 91 "sql.y"
 { yymsp[-1].minor.yy748 = yymsp[0].minor.yy748; }
+#line 5650 "sql.c"
         break;
       case 27: /* white_list_opt ::= */
       case 200: /* specific_cols_opt ::= */ yytestcase(yyruleno==200);
@@ -5597,89 +5658,137 @@ static YYACTIONTYPE yy_reduce(
       case 671: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==671);
       case 701: /* group_by_clause_opt ::= */ yytestcase(yyruleno==701);
       case 721: /* order_by_clause_opt ::= */ yytestcase(yyruleno==721);
+#line 95 "sql.y"
 { yymsp[1].minor.yy748 = NULL; }
+#line 5663 "sql.c"
         break;
       case 28: /* white_list_opt ::= white_list */
       case 239: /* tags_def_opt ::= tags_def */ yytestcase(yyruleno==239);
       case 409: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==409);
       case 579: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==579);
+#line 96 "sql.y"
 { yylhsminor.yy748 = yymsp[0].minor.yy748; }
+#line 5671 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 29: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt white_list_opt */
+#line 100 "sql.y"
 {
                                                                                     pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-4].minor.yy649, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy663);
                                                                                     pCxt->pRootNode = addCreateUserStmtWhiteList(pCxt, pCxt->pRootNode, yymsp[0].minor.yy748);
                                                                                   }
+#line 5680 "sql.c"
         break;
       case 30: /* cmd ::= ALTER USER user_name PASS NK_STRING */
+#line 104 "sql.y"
 { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); }
+#line 5685 "sql.c"
         break;
       case 31: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */
+#line 105 "sql.y"
 { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); }
+#line 5690 "sql.c"
         break;
       case 32: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */
+#line 106 "sql.y"
 { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); }
+#line 5695 "sql.c"
         break;
       case 33: /* cmd ::= ALTER USER user_name ADD white_list */
+#line 107 "sql.y"
 { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_ADD_WHITE_LIST, yymsp[0].minor.yy748); }
+#line 5700 "sql.c"
         break;
       case 34: /* cmd ::= ALTER USER user_name DROP white_list */
+#line 108 "sql.y"
 { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_DROP_WHITE_LIST, yymsp[0].minor.yy748); }
+#line 5705 "sql.c"
         break;
       case 35: /* cmd ::= DROP USER user_name */
+#line 109 "sql.y"
 { pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy649); }
+#line 5710 "sql.c"
         break;
       case 36: /* sysinfo_opt ::= */
+#line 113 "sql.y"
 { yymsp[1].minor.yy663 = 1; }
+#line 5715 "sql.c"
         break;
       case 37: /* sysinfo_opt ::= SYSINFO NK_INTEGER */
+#line 114 "sql.y"
 { yymsp[-1].minor.yy663 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); }
+#line 5720 "sql.c"
         break;
       case 38: /* cmd ::= GRANT privileges ON priv_level with_opt TO user_name */
+#line 117 "sql.y"
 { pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy941, &yymsp[-3].minor.yy781, &yymsp[0].minor.yy649, yymsp[-2].minor.yy600); }
+#line 5725 "sql.c"
         break;
       case 39: /* cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */
+#line 118 "sql.y"
 { pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy941, &yymsp[-3].minor.yy781, &yymsp[0].minor.yy649, yymsp[-2].minor.yy600); }
+#line 5730 "sql.c"
         break;
       case 40: /* privileges ::= ALL */
+#line 122 "sql.y"
 { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_ALL; }
+#line 5735 "sql.c"
         break;
       case 41: /* privileges ::= priv_type_list */
       case 43: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==43);
+#line 123 "sql.y"
 { yylhsminor.yy941 = yymsp[0].minor.yy941; }
+#line 5741 "sql.c"
   yymsp[0].minor.yy941 = yylhsminor.yy941;
         break;
       case 42: /* privileges ::= SUBSCRIBE */
+#line 124 "sql.y"
 { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_SUBSCRIBE; }
+#line 5747 "sql.c"
         break;
       case 44: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */
+#line 129 "sql.y"
 { yylhsminor.yy941 = yymsp[-2].minor.yy941 | yymsp[0].minor.yy941; }
+#line 5752 "sql.c"
   yymsp[-2].minor.yy941 = yylhsminor.yy941;
         break;
       case 45: /* priv_type ::= READ */
+#line 133 "sql.y"
 { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_READ; }
+#line 5758 "sql.c"
         break;
       case 46: /* priv_type ::= WRITE */
+#line 134 "sql.y"
 { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_WRITE; }
+#line 5763 "sql.c"
         break;
       case 47: /* priv_type ::= ALTER */
+#line 135 "sql.y"
 { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_ALTER; }
+#line 5768 "sql.c"
         break;
       case 48: /* priv_level ::= NK_STAR NK_DOT NK_STAR */
+#line 139 "sql.y"
 { yylhsminor.yy781.first = yymsp[-2].minor.yy0; yylhsminor.yy781.second = yymsp[0].minor.yy0; }
+#line 5773 "sql.c"
   yymsp[-2].minor.yy781 = yylhsminor.yy781;
         break;
       case 49: /* priv_level ::= db_name NK_DOT NK_STAR */
+#line 140 "sql.y"
 { yylhsminor.yy781.first = yymsp[-2].minor.yy649; yylhsminor.yy781.second = yymsp[0].minor.yy0; }
+#line 5779 "sql.c"
   yymsp[-2].minor.yy781 = yylhsminor.yy781;
         break;
       case 50: /* priv_level ::= db_name NK_DOT table_name */
+#line 141 "sql.y"
 { yylhsminor.yy781.first = yymsp[-2].minor.yy649; yylhsminor.yy781.second = yymsp[0].minor.yy649; }
+#line 5785 "sql.c"
   yymsp[-2].minor.yy781 = yylhsminor.yy781;
         break;
       case 51: /* priv_level ::= topic_name */
+#line 142 "sql.y"
 { yylhsminor.yy781.first = yymsp[0].minor.yy649; yylhsminor.yy781.second = nil_token; }
+#line 5791 "sql.c"
   yymsp[0].minor.yy781 = yylhsminor.yy781;
         break;
       case 52: /* with_opt ::= */
@@ -5701,50 +5810,78 @@ static YYACTIONTYPE yy_reduce(
       case 710: /* every_opt ::= */ yytestcase(yyruleno==710);
       case 723: /* slimit_clause_opt ::= */ yytestcase(yyruleno==723);
       case 727: /* limit_clause_opt ::= */ yytestcase(yyruleno==727);
+#line 144 "sql.y"
 { yymsp[1].minor.yy600 = NULL; }
+#line 5815 "sql.c"
         break;
       case 53: /* with_opt ::= WITH search_condition */
       case 620: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==620);
       case 647: /* join_on_clause_opt ::= ON search_condition */ yytestcase(yyruleno==647);
       case 670: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==670);
       case 706: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==706);
+#line 145 "sql.y"
 { yymsp[-1].minor.yy600 = yymsp[0].minor.yy600; }
+#line 5824 "sql.c"
         break;
       case 54: /* cmd ::= CREATE ENCRYPT_KEY NK_STRING */
+#line 148 "sql.y"
 { pCxt->pRootNode = createEncryptKeyStmt(pCxt, &yymsp[0].minor.yy0); }
+#line 5829 "sql.c"
         break;
       case 55: /* cmd ::= CREATE DNODE dnode_endpoint */
+#line 151 "sql.y"
 { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy649, NULL); }
+#line 5834 "sql.c"
         break;
       case 56: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */
+#line 152 "sql.y"
 { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0); }
+#line 5839 "sql.c"
         break;
       case 57: /* cmd ::= DROP DNODE NK_INTEGER force_opt */
+#line 153 "sql.y"
 { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy705, false); }
+#line 5844 "sql.c"
         break;
       case 58: /* cmd ::= DROP DNODE dnode_endpoint force_opt */
+#line 154 "sql.y"
 { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy705, false); }
+#line 5849 "sql.c"
         break;
       case 59: /* cmd ::= DROP DNODE NK_INTEGER unsafe_opt */
+#line 155 "sql.y"
 { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, false, yymsp[0].minor.yy705); }
+#line 5854 "sql.c"
         break;
       case 60: /* cmd ::= DROP DNODE dnode_endpoint unsafe_opt */
+#line 156 "sql.y"
 { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy649, false, yymsp[0].minor.yy705); }
+#line 5859 "sql.c"
         break;
       case 61: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */
+#line 157 "sql.y"
 { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); }
+#line 5864 "sql.c"
         break;
       case 62: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */
+#line 158 "sql.y"
 { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); }
+#line 5869 "sql.c"
         break;
       case 63: /* cmd ::= ALTER ALL DNODES NK_STRING */
+#line 159 "sql.y"
 { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); }
+#line 5874 "sql.c"
         break;
       case 64: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */
+#line 160 "sql.y"
 { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); }
+#line 5879 "sql.c"
         break;
       case 65: /* cmd ::= RESTORE DNODE NK_INTEGER */
+#line 161 "sql.y"
 { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_DNODE_STMT, &yymsp[0].minor.yy0); }
+#line 5884 "sql.c"
         break;
       case 66: /* dnode_endpoint ::= NK_STRING */
       case 67: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==67);
@@ -5780,7 +5917,9 @@ static YYACTIONTYPE yy_reduce(
       case 575: /* star_func ::= FIRST */ yytestcase(yyruleno==575);
       case 576: /* star_func ::= LAST */ yytestcase(yyruleno==576);
       case 577: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==577);
+#line 165 "sql.y"
 { yylhsminor.yy649 = yymsp[0].minor.yy0; }
+#line 5922 "sql.c"
   yymsp[0].minor.yy649 = yylhsminor.yy649;
         break;
       case 69: /* force_opt ::= */
@@ -5792,7 +5931,9 @@ static YYACTIONTYPE yy_reduce(
       case 422: /* ignore_opt ::= */ yytestcase(yyruleno==422);
       case 657: /* tag_mode_opt ::= */ yytestcase(yyruleno==657);
       case 659: /* set_quantifier_opt ::= */ yytestcase(yyruleno==659);
+#line 171 "sql.y"
 { yymsp[1].minor.yy705 = false; }
+#line 5936 "sql.c"
         break;
       case 70: /* force_opt ::= FORCE */
       case 71: /* unsafe_opt ::= UNSAFE */ yytestcase(yyruleno==71);
@@ -5800,318 +5941,486 @@ static YYACTIONTYPE yy_reduce(
       case 386: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==386);
       case 658: /* tag_mode_opt ::= TAGS */ yytestcase(yyruleno==658);
       case 660: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==660);
+#line 172 "sql.y"
 { yymsp[0].minor.yy705 = true; }
+#line 5946 "sql.c"
         break;
       case 72: /* cmd ::= ALTER CLUSTER NK_STRING */
+#line 179 "sql.y"
 { pCxt->pRootNode = createAlterClusterStmt(pCxt, &yymsp[0].minor.yy0, NULL); }
+#line 5951 "sql.c"
         break;
       case 73: /* cmd ::= ALTER CLUSTER NK_STRING NK_STRING */
+#line 180 "sql.y"
 { pCxt->pRootNode = createAlterClusterStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); }
+#line 5956 "sql.c"
         break;
       case 74: /* cmd ::= ALTER LOCAL NK_STRING */
+#line 183 "sql.y"
 { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); }
+#line 5961 "sql.c"
         break;
       case 75: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */
+#line 184 "sql.y"
 { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); }
+#line 5966 "sql.c"
         break;
       case 76: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */
+#line 187 "sql.y"
 { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_QNODE_STMT, &yymsp[0].minor.yy0); }
+#line 5971 "sql.c"
         break;
       case 77: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */
+#line 188 "sql.y"
 { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_QNODE_STMT, &yymsp[0].minor.yy0); }
+#line 5976 "sql.c"
         break;
       case 78: /* cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */
+#line 189 "sql.y"
 { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_QNODE_STMT, &yymsp[0].minor.yy0); }
+#line 5981 "sql.c"
         break;
       case 79: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */
+#line 192 "sql.y"
 { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_BNODE_STMT, &yymsp[0].minor.yy0); }
+#line 5986 "sql.c"
         break;
       case 80: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */
+#line 193 "sql.y"
 { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_BNODE_STMT, &yymsp[0].minor.yy0); }
+#line 5991 "sql.c"
         break;
       case 81: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */
+#line 196 "sql.y"
 { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_SNODE_STMT, &yymsp[0].minor.yy0); }
+#line 5996 "sql.c"
         break;
       case 82: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */
+#line 197 "sql.y"
 { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_SNODE_STMT, &yymsp[0].minor.yy0); }
+#line 6001 "sql.c"
         break;
       case 83: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */
+#line 200 "sql.y"
 { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &yymsp[0].minor.yy0); }
+#line 6006 "sql.c"
         break;
       case 84: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */
+#line 201 "sql.y"
 { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); }
+#line 6011 "sql.c"
         break;
       case 85: /* cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */
+#line 202 "sql.y"
 { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_MNODE_STMT, &yymsp[0].minor.yy0); }
+#line 6016 "sql.c"
         break;
       case 86: /* cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */
+#line 205 "sql.y"
 { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_VNODE_STMT, &yymsp[0].minor.yy0); }
+#line 6021 "sql.c"
         break;
       case 87: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */
+#line 208 "sql.y"
 { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy705, &yymsp[-1].minor.yy649, yymsp[0].minor.yy600); }
+#line 6026 "sql.c"
         break;
       case 88: /* cmd ::= DROP DATABASE exists_opt db_name */
+#line 209 "sql.y"
 { pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); }
+#line 6031 "sql.c"
         break;
       case 89: /* cmd ::= USE db_name */
+#line 210 "sql.y"
 { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy649); }
+#line 6036 "sql.c"
         break;
       case 90: /* cmd ::= ALTER DATABASE db_name alter_db_options */
+#line 211 "sql.y"
 { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy600); }
+#line 6041 "sql.c"
         break;
       case 91: /* cmd ::= FLUSH DATABASE db_name */
+#line 212 "sql.y"
 { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy649); }
+#line 6046 "sql.c"
         break;
       case 92: /* cmd ::= TRIM DATABASE db_name speed_opt */
+#line 213 "sql.y"
 { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy756); }
+#line 6051 "sql.c"
         break;
       case 93: /* cmd ::= S3MIGRATE DATABASE db_name */
+#line 214 "sql.y"
 { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &yymsp[0].minor.yy649); }
+#line 6056 "sql.c"
         break;
       case 94: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */
+#line 215 "sql.y"
 { pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy649, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 6061 "sql.c"
         break;
       case 95: /* not_exists_opt ::= IF NOT EXISTS */
+#line 219 "sql.y"
 { yymsp[-2].minor.yy705 = true; }
+#line 6066 "sql.c"
         break;
       case 97: /* exists_opt ::= IF EXISTS */
       case 392: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==392);
       case 423: /* ignore_opt ::= IGNORE UNTREATED */ yytestcase(yyruleno==423);
+#line 224 "sql.y"
 { yymsp[-1].minor.yy705 = true; }
+#line 6073 "sql.c"
         break;
       case 99: /* db_options ::= */
+#line 227 "sql.y"
 { yymsp[1].minor.yy600 = createDefaultDatabaseOptions(pCxt); }
+#line 6078 "sql.c"
         break;
       case 100: /* db_options ::= db_options BUFFER NK_INTEGER */
+#line 228 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); }
+#line 6083 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 101: /* db_options ::= db_options CACHEMODEL NK_STRING */
+#line 229 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); }
+#line 6089 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 102: /* db_options ::= db_options CACHESIZE NK_INTEGER */
+#line 230 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); }
+#line 6095 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 103: /* db_options ::= db_options COMP NK_INTEGER */
+#line 231 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_COMP, &yymsp[0].minor.yy0); }
+#line 6101 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 104: /* db_options ::= db_options DURATION NK_INTEGER */
       case 105: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==105);
+#line 232 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_DAYS, &yymsp[0].minor.yy0); }
+#line 6108 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 106: /* db_options ::= db_options MAXROWS NK_INTEGER */
+#line 234 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); }
+#line 6114 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 107: /* db_options ::= db_options MINROWS NK_INTEGER */
+#line 235 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); }
+#line 6120 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 108: /* db_options ::= db_options KEEP integer_list */
       case 109: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==109);
+#line 236 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_KEEP, yymsp[0].minor.yy748); }
+#line 6127 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 110: /* db_options ::= db_options PAGES NK_INTEGER */
+#line 238 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGES, &yymsp[0].minor.yy0); }
+#line 6133 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 111: /* db_options ::= db_options PAGESIZE NK_INTEGER */
+#line 239 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); }
+#line 6139 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 112: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */
+#line 240 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); }
+#line 6145 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 113: /* db_options ::= db_options PRECISION NK_STRING */
+#line 241 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); }
+#line 6151 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 114: /* db_options ::= db_options REPLICA NK_INTEGER */
+#line 242 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); }
+#line 6157 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 115: /* db_options ::= db_options VGROUPS NK_INTEGER */
+#line 244 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); }
+#line 6163 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 116: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */
+#line 245 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); }
+#line 6169 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 117: /* db_options ::= db_options RETENTIONS retention_list */
+#line 246 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_RETENTIONS, yymsp[0].minor.yy748); }
+#line 6175 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 118: /* db_options ::= db_options SCHEMALESS NK_INTEGER */
+#line 247 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); }
+#line 6181 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 119: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */
+#line 248 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL, &yymsp[0].minor.yy0); }
+#line 6187 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 120: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */
+#line 249 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); }
+#line 6193 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 121: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */
+#line 250 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); }
+#line 6199 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 122: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */
+#line 251 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &t);
                                                                                   }
+#line 6209 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 123: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */
+#line 256 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); }
+#line 6215 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 124: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */
+#line 257 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &t);
                                                                                   }
+#line 6225 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 125: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */
+#line 262 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); }
+#line 6231 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 126: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */
+#line 263 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); }
+#line 6237 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 127: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */
+#line 264 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); }
+#line 6243 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 128: /* db_options ::= db_options TABLE_PREFIX signed */
+#line 265 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy600); }
+#line 6249 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 129: /* db_options ::= db_options TABLE_SUFFIX signed */
+#line 266 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy600); }
+#line 6255 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 130: /* db_options ::= db_options S3_CHUNKSIZE NK_INTEGER */
+#line 267 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_S3_CHUNKSIZE, &yymsp[0].minor.yy0); }
+#line 6261 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 131: /* db_options ::= db_options S3_KEEPLOCAL NK_INTEGER */
       case 132: /* db_options ::= db_options S3_KEEPLOCAL NK_VARIABLE */ yytestcase(yyruleno==132);
+#line 268 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_S3_KEEPLOCAL, &yymsp[0].minor.yy0); }
+#line 6268 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 133: /* db_options ::= db_options S3_COMPACT NK_INTEGER */
+#line 270 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_S3_COMPACT, &yymsp[0].minor.yy0); }
+#line 6274 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 134: /* db_options ::= db_options KEEP_TIME_OFFSET NK_INTEGER */
+#line 271 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_KEEP_TIME_OFFSET, &yymsp[0].minor.yy0); }
+#line 6280 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 135: /* db_options ::= db_options ENCRYPT_ALGORITHM NK_STRING */
+#line 272 "sql.y"
 { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_ENCRYPT_ALGORITHM, &yymsp[0].minor.yy0); }
+#line 6286 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 136: /* alter_db_options ::= alter_db_option */
+#line 274 "sql.y"
 { yylhsminor.yy600 = createAlterDatabaseOptions(pCxt); yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yylhsminor.yy600, &yymsp[0].minor.yy145); }
+#line 6292 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 137: /* alter_db_options ::= alter_db_options alter_db_option */
+#line 275 "sql.y"
 { yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy600, &yymsp[0].minor.yy145); }
+#line 6298 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 138: /* alter_db_option ::= BUFFER NK_INTEGER */
+#line 279 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6304 "sql.c"
         break;
       case 139: /* alter_db_option ::= CACHEMODEL NK_STRING */
+#line 280 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6309 "sql.c"
         break;
       case 140: /* alter_db_option ::= CACHESIZE NK_INTEGER */
+#line 281 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6314 "sql.c"
         break;
       case 141: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */
+#line 282 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6319 "sql.c"
         break;
       case 142: /* alter_db_option ::= KEEP integer_list */
       case 143: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==143);
+#line 283 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_KEEP; yymsp[-1].minor.yy145.pList = yymsp[0].minor.yy748; }
+#line 6325 "sql.c"
         break;
       case 144: /* alter_db_option ::= PAGES NK_INTEGER */
+#line 285 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_PAGES; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6330 "sql.c"
         break;
       case 145: /* alter_db_option ::= REPLICA NK_INTEGER */
+#line 286 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6335 "sql.c"
         break;
       case 146: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */
+#line 288 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_WAL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6340 "sql.c"
         break;
       case 147: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */
+#line 289 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6345 "sql.c"
         break;
       case 148: /* alter_db_option ::= MINROWS NK_INTEGER */
+#line 290 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6350 "sql.c"
         break;
       case 149: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */
+#line 291 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6355 "sql.c"
         break;
       case 150: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */
+#line 292 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yymsp[-2].minor.yy145.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy145.val = t;
                                                                                   }
+#line 6364 "sql.c"
         break;
       case 151: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */
+#line 297 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6369 "sql.c"
         break;
       case 152: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */
+#line 298 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yymsp[-2].minor.yy145.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy145.val = t;
                                                                                   }
+#line 6378 "sql.c"
         break;
       case 153: /* alter_db_option ::= S3_KEEPLOCAL NK_INTEGER */
       case 154: /* alter_db_option ::= S3_KEEPLOCAL NK_VARIABLE */ yytestcase(yyruleno==154);
+#line 303 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_S3_KEEPLOCAL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6384 "sql.c"
         break;
       case 155: /* alter_db_option ::= S3_COMPACT NK_INTEGER */
+#line 305 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_S3_COMPACT, yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6389 "sql.c"
         break;
       case 156: /* alter_db_option ::= KEEP_TIME_OFFSET NK_INTEGER */
+#line 306 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_KEEP_TIME_OFFSET; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6394 "sql.c"
         break;
       case 157: /* alter_db_option ::= ENCRYPT_ALGORITHM NK_STRING */
+#line 307 "sql.y"
 { yymsp[-1].minor.yy145.type = DB_OPTION_ENCRYPT_ALGORITHM; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6399 "sql.c"
         break;
       case 158: /* integer_list ::= NK_INTEGER */
+#line 311 "sql.y"
 { yylhsminor.yy748 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); }
+#line 6404 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 159: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */
       case 437: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==437);
+#line 312 "sql.y"
 { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); }
+#line 6411 "sql.c"
   yymsp[-2].minor.yy748 = yylhsminor.yy748;
         break;
       case 160: /* variable_list ::= NK_VARIABLE */
+#line 316 "sql.y"
 { yylhsminor.yy748 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); }
+#line 6417 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 161: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */
+#line 317 "sql.y"
 { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); }
+#line 6423 "sql.c"
   yymsp[-2].minor.yy748 = yylhsminor.yy748;
         break;
       case 162: /* retention_list ::= retention */
@@ -6131,7 +6440,9 @@ static YYACTIONTYPE yy_reduce(
       case 662: /* select_list ::= select_item */ yytestcase(yyruleno==662);
       case 673: /* partition_list ::= partition_item */ yytestcase(yyruleno==673);
       case 734: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==734);
+#line 321 "sql.y"
 { yylhsminor.yy748 = createNodeList(pCxt, yymsp[0].minor.yy600); }
+#line 6445 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 163: /* retention_list ::= retention_list NK_COMMA retention */
@@ -6149,750 +6460,1178 @@ static YYACTIONTYPE yy_reduce(
       case 663: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==663);
       case 674: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==674);
       case 735: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==735);
+#line 322 "sql.y"
 { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, yymsp[0].minor.yy600); }
+#line 6465 "sql.c"
   yymsp[-2].minor.yy748 = yylhsminor.yy748;
         break;
       case 164: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */
       case 165: /* retention ::= NK_MINUS NK_COLON NK_VARIABLE */ yytestcase(yyruleno==165);
+#line 324 "sql.y"
 { yylhsminor.yy600 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); }
+#line 6472 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 166: /* speed_opt ::= */
       case 387: /* bufsize_opt ::= */ yytestcase(yyruleno==387);
+#line 329 "sql.y"
 { yymsp[1].minor.yy756 = 0; }
+#line 6479 "sql.c"
         break;
       case 167: /* speed_opt ::= BWLIMIT NK_INTEGER */
       case 388: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==388);
+#line 330 "sql.y"
 { yymsp[-1].minor.yy756 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); }
+#line 6485 "sql.c"
         break;
       case 169: /* start_opt ::= START WITH NK_INTEGER */
       case 173: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==173);
+#line 333 "sql.y"
 { yymsp[-2].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); }
+#line 6491 "sql.c"
         break;
       case 170: /* start_opt ::= START WITH NK_STRING */
       case 174: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==174);
+#line 334 "sql.y"
 { yymsp[-2].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); }
+#line 6497 "sql.c"
         break;
       case 171: /* start_opt ::= START WITH TIMESTAMP NK_STRING */
       case 175: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==175);
+#line 335 "sql.y"
 { yymsp[-3].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); }
+#line 6503 "sql.c"
         break;
       case 176: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */
       case 178: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==178);
+#line 344 "sql.y"
 { pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy705, yymsp[-5].minor.yy600, yymsp[-3].minor.yy748, yymsp[-1].minor.yy748, yymsp[0].minor.yy600); }
+#line 6509 "sql.c"
         break;
       case 177: /* cmd ::= CREATE TABLE multi_create_clause */
+#line 345 "sql.y"
 { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy748); }
+#line 6514 "sql.c"
         break;
       case 179: /* cmd ::= DROP TABLE multi_drop_clause */
+#line 348 "sql.y"
 { pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy748); }
+#line 6519 "sql.c"
         break;
       case 180: /* cmd ::= DROP STABLE exists_opt full_table_name */
+#line 349 "sql.y"
 { pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); }
+#line 6524 "sql.c"
         break;
       case 181: /* cmd ::= ALTER TABLE alter_table_clause */
       case 439: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==439);
       case 440: /* cmd ::= insert_query */ yytestcase(yyruleno==440);
+#line 351 "sql.y"
 { pCxt->pRootNode = yymsp[0].minor.yy600; }
+#line 6531 "sql.c"
         break;
       case 182: /* cmd ::= ALTER STABLE alter_table_clause */
+#line 352 "sql.y"
 { pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy600); }
+#line 6536 "sql.c"
         break;
       case 183: /* alter_table_clause ::= full_table_name alter_table_options */
+#line 354 "sql.y"
 { yylhsminor.yy600 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 6541 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
-      case 184: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */
-{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); }
-  yymsp[-4].minor.yy600 = yylhsminor.yy600;
+      case 184: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options */
+#line 356 "sql.y"
+{ yylhsminor.yy600 = createAlterTableAddModifyColOptions2(pCxt, yymsp[-5].minor.yy600, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-2].minor.yy649, yymsp[-1].minor.yy400, yymsp[0].minor.yy600); }
+#line 6547 "sql.c"
+  yymsp[-5].minor.yy600 = yylhsminor.yy600;
         break;
       case 185: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */
+#line 357 "sql.y"
 { yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy649); }
+#line 6553 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 186: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */
+#line 359 "sql.y"
 { yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); }
+#line 6559 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 187: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options */
+#line 361 "sql.y"
 { yylhsminor.yy600 = createAlterTableAddModifyColOptions(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS, &yymsp[-1].minor.yy649, yymsp[0].minor.yy600); }
+#line 6565 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 188: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */
+#line 363 "sql.y"
 { yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); }
+#line 6571 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 189: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */
+#line 365 "sql.y"
 { yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); }
+#line 6577 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 190: /* alter_table_clause ::= full_table_name DROP TAG column_name */
+#line 366 "sql.y"
 { yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy649); }
+#line 6583 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 191: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */
+#line 368 "sql.y"
 { yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); }
+#line 6589 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 192: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */
+#line 370 "sql.y"
 { yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); }
+#line 6595 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 193: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ tags_literal */
+#line 372 "sql.y"
 { yylhsminor.yy600 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy600, &yymsp[-2].minor.yy649, yymsp[0].minor.yy600); }
+#line 6601 "sql.c"
   yymsp[-5].minor.yy600 = yylhsminor.yy600;
         break;
       case 195: /* multi_create_clause ::= multi_create_clause create_subtable_clause */
       case 587: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==587);
+#line 377 "sql.y"
 { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-1].minor.yy748, yymsp[0].minor.yy600); }
+#line 6608 "sql.c"
   yymsp[-1].minor.yy748 = yylhsminor.yy748;
         break;
       case 196: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP tags_literal_list NK_RP table_options */
+#line 381 "sql.y"
 { yylhsminor.yy600 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy705, yymsp[-8].minor.yy600, yymsp[-6].minor.yy600, yymsp[-5].minor.yy748, yymsp[-2].minor.yy748, yymsp[0].minor.yy600); }
+#line 6614 "sql.c"
   yymsp[-9].minor.yy600 = yylhsminor.yy600;
         break;
       case 199: /* drop_table_clause ::= exists_opt full_table_name */
+#line 388 "sql.y"
 { yylhsminor.yy600 = createDropTableClause(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); }
+#line 6620 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 201: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */
       case 402: /* col_list_opt ::= NK_LP column_stream_def_list NK_RP */ yytestcase(yyruleno==402);
+#line 393 "sql.y"
 { yymsp[-2].minor.yy748 = yymsp[-1].minor.yy748; }
+#line 6627 "sql.c"
         break;
       case 202: /* full_table_name ::= table_name */
       case 343: /* full_tsma_name ::= tsma_name */ yytestcase(yyruleno==343);
+#line 395 "sql.y"
 { yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy649, NULL); }
+#line 6633 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 203: /* full_table_name ::= db_name NK_DOT table_name */
       case 344: /* full_tsma_name ::= db_name NK_DOT tsma_name */ yytestcase(yyruleno==344);
+#line 396 "sql.y"
 { yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649, NULL); }
+#line 6640 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 206: /* tag_def ::= column_name type_name */
+#line 402 "sql.y"
 { yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400, NULL); }
+#line 6646 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 209: /* column_def ::= column_name type_name column_options */
+#line 410 "sql.y"
 { yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-2].minor.yy649, yymsp[-1].minor.yy400, yymsp[0].minor.yy600); }
+#line 6652 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 210: /* type_name ::= BOOL */
+#line 414 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BOOL); }
+#line 6658 "sql.c"
         break;
       case 211: /* type_name ::= TINYINT */
+#line 415 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_TINYINT); }
+#line 6663 "sql.c"
         break;
       case 212: /* type_name ::= SMALLINT */
+#line 416 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_SMALLINT); }
+#line 6668 "sql.c"
         break;
       case 213: /* type_name ::= INT */
       case 214: /* type_name ::= INTEGER */ yytestcase(yyruleno==214);
+#line 417 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_INT); }
+#line 6674 "sql.c"
         break;
       case 215: /* type_name ::= BIGINT */
+#line 419 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BIGINT); }
+#line 6679 "sql.c"
         break;
       case 216: /* type_name ::= FLOAT */
+#line 420 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_FLOAT); }
+#line 6684 "sql.c"
         break;
       case 217: /* type_name ::= DOUBLE */
+#line 421 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_DOUBLE); }
+#line 6689 "sql.c"
         break;
       case 218: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */
+#line 422 "sql.y"
 { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); }
+#line 6694 "sql.c"
         break;
       case 219: /* type_name ::= TIMESTAMP */
+#line 423 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); }
+#line 6699 "sql.c"
         break;
       case 220: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */
+#line 424 "sql.y"
 { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); }
+#line 6704 "sql.c"
         break;
       case 221: /* type_name ::= TINYINT UNSIGNED */
+#line 425 "sql.y"
 { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UTINYINT); }
+#line 6709 "sql.c"
         break;
       case 222: /* type_name ::= SMALLINT UNSIGNED */
+#line 426 "sql.y"
 { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_USMALLINT); }
+#line 6714 "sql.c"
         break;
       case 223: /* type_name ::= INT UNSIGNED */
+#line 427 "sql.y"
 { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UINT); }
+#line 6719 "sql.c"
         break;
       case 224: /* type_name ::= BIGINT UNSIGNED */
+#line 428 "sql.y"
 { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UBIGINT); }
+#line 6724 "sql.c"
         break;
       case 225: /* type_name ::= JSON */
+#line 429 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_JSON); }
+#line 6729 "sql.c"
         break;
       case 226: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */
+#line 430 "sql.y"
 { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); }
+#line 6734 "sql.c"
         break;
       case 227: /* type_name ::= MEDIUMBLOB */
+#line 431 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); }
+#line 6739 "sql.c"
         break;
       case 228: /* type_name ::= BLOB */
+#line 432 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BLOB); }
+#line 6744 "sql.c"
         break;
       case 229: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */
+#line 433 "sql.y"
 { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); }
+#line 6749 "sql.c"
         break;
       case 230: /* type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */
+#line 434 "sql.y"
 { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &yymsp[-1].minor.yy0); }
+#line 6754 "sql.c"
         break;
       case 231: /* type_name ::= DECIMAL */
+#line 435 "sql.y"
 { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); }
+#line 6759 "sql.c"
         break;
       case 232: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */
+#line 436 "sql.y"
 { yymsp[-3].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); }
+#line 6764 "sql.c"
         break;
       case 233: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */
+#line 437 "sql.y"
 { yymsp[-5].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); }
+#line 6769 "sql.c"
         break;
       case 234: /* type_name_default_len ::= BINARY */
+#line 441 "sql.y"
 { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, NULL); }
+#line 6774 "sql.c"
         break;
       case 235: /* type_name_default_len ::= NCHAR */
+#line 442 "sql.y"
 { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, NULL); }
+#line 6779 "sql.c"
         break;
       case 236: /* type_name_default_len ::= VARCHAR */
+#line 443 "sql.y"
 { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, NULL); }
+#line 6784 "sql.c"
         break;
       case 237: /* type_name_default_len ::= VARBINARY */
+#line 444 "sql.y"
 { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, NULL); }
+#line 6789 "sql.c"
         break;
       case 240: /* tags_def ::= TAGS NK_LP tag_def_list NK_RP */
       case 410: /* tag_def_or_ref_opt ::= TAGS NK_LP column_stream_def_list NK_RP */ yytestcase(yyruleno==410);
+#line 453 "sql.y"
 { yymsp[-3].minor.yy748 = yymsp[-1].minor.yy748; }
+#line 6795 "sql.c"
         break;
       case 241: /* table_options ::= */
+#line 455 "sql.y"
 { yymsp[1].minor.yy600 = createDefaultTableOptions(pCxt); }
+#line 6800 "sql.c"
         break;
       case 242: /* table_options ::= table_options COMMENT NK_STRING */
+#line 456 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); }
+#line 6805 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 243: /* table_options ::= table_options MAX_DELAY duration_list */
+#line 457 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy748); }
+#line 6811 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 244: /* table_options ::= table_options WATERMARK duration_list */
+#line 458 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy748); }
+#line 6817 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 245: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */
+#line 459 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy748); }
+#line 6823 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 246: /* table_options ::= table_options TTL NK_INTEGER */
+#line 460 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); }
+#line 6829 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 247: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */
+#line 461 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_SMA, yymsp[-1].minor.yy748); }
+#line 6835 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 248: /* table_options ::= table_options DELETE_MARK duration_list */
+#line 462 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy748); }
+#line 6841 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 249: /* alter_table_options ::= alter_table_option */
+#line 464 "sql.y"
 { yylhsminor.yy600 = createAlterTableOptions(pCxt); yylhsminor.yy600 = setTableOption(pCxt, yylhsminor.yy600, yymsp[0].minor.yy145.type, &yymsp[0].minor.yy145.val); }
+#line 6847 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 250: /* alter_table_options ::= alter_table_options alter_table_option */
+#line 465 "sql.y"
 { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy145.type, &yymsp[0].minor.yy145.val); }
+#line 6853 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 251: /* alter_table_option ::= COMMENT NK_STRING */
+#line 469 "sql.y"
 { yymsp[-1].minor.yy145.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6859 "sql.c"
         break;
       case 252: /* alter_table_option ::= TTL NK_INTEGER */
+#line 470 "sql.y"
 { yymsp[-1].minor.yy145.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; }
+#line 6864 "sql.c"
         break;
       case 253: /* duration_list ::= duration_literal */
       case 539: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==539);
+#line 474 "sql.y"
 { yylhsminor.yy748 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); }
+#line 6870 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 254: /* duration_list ::= duration_list NK_COMMA duration_literal */
       case 540: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==540);
+#line 475 "sql.y"
 { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); }
+#line 6877 "sql.c"
   yymsp[-2].minor.yy748 = yylhsminor.yy748;
         break;
       case 257: /* rollup_func_name ::= function_name */
+#line 482 "sql.y"
 { yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy649, NULL); }
+#line 6883 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 258: /* rollup_func_name ::= FIRST */
       case 259: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==259);
       case 332: /* tag_item ::= QTAGS */ yytestcase(yyruleno==332);
+#line 483 "sql.y"
 { yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); }
+#line 6891 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 262: /* col_name ::= column_name */
       case 333: /* tag_item ::= column_name */ yytestcase(yyruleno==333);
+#line 491 "sql.y"
 { yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy649); }
+#line 6898 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 263: /* cmd ::= SHOW DNODES */
+#line 494 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); }
+#line 6904 "sql.c"
         break;
       case 264: /* cmd ::= SHOW USERS */
+#line 495 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); }
+#line 6909 "sql.c"
         break;
       case 265: /* cmd ::= SHOW USER PRIVILEGES */
+#line 496 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); }
+#line 6914 "sql.c"
         break;
       case 266: /* cmd ::= SHOW db_kind_opt DATABASES */
+#line 497 "sql.y"
 {
                                                                                     pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT);
                                                                                     setShowKind(pCxt, pCxt->pRootNode, yymsp[-1].minor.yy245);
                                                                                   }
+#line 6922 "sql.c"
         break;
       case 267: /* cmd ::= SHOW table_kind_db_name_cond_opt TABLES like_pattern_opt */
+#line 501 "sql.y"
 {
                                                                                     pCxt->pRootNode = createShowTablesStmt(pCxt, yymsp[-2].minor.yy125, yymsp[0].minor.yy600, OP_TYPE_LIKE);
                                                                                   }
+#line 6929 "sql.c"
         break;
       case 268: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */
+#line 504 "sql.y"
 { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); }
+#line 6934 "sql.c"
         break;
       case 269: /* cmd ::= SHOW db_name_cond_opt VGROUPS */
+#line 505 "sql.y"
 { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy600, NULL, OP_TYPE_LIKE); }
+#line 6939 "sql.c"
         break;
       case 270: /* cmd ::= SHOW MNODES */
+#line 506 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); }
+#line 6944 "sql.c"
         break;
       case 271: /* cmd ::= SHOW QNODES */
+#line 508 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); }
+#line 6949 "sql.c"
         break;
       case 272: /* cmd ::= SHOW ARBGROUPS */
+#line 509 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ARBGROUPS_STMT); }
+#line 6954 "sql.c"
         break;
       case 273: /* cmd ::= SHOW FUNCTIONS */
+#line 510 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); }
+#line 6959 "sql.c"
         break;
       case 274: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */
+#line 511 "sql.y"
 { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); }
+#line 6964 "sql.c"
         break;
       case 275: /* cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */
+#line 512 "sql.y"
 { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy649), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649), OP_TYPE_EQUAL); }
+#line 6969 "sql.c"
         break;
       case 276: /* cmd ::= SHOW STREAMS */
+#line 513 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); }
+#line 6974 "sql.c"
         break;
       case 277: /* cmd ::= SHOW ACCOUNTS */
+#line 514 "sql.y"
 { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
+#line 6979 "sql.c"
         break;
       case 278: /* cmd ::= SHOW APPS */
+#line 515 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); }
+#line 6984 "sql.c"
         break;
       case 279: /* cmd ::= SHOW CONNECTIONS */
+#line 516 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); }
+#line 6989 "sql.c"
         break;
       case 280: /* cmd ::= SHOW LICENCES */
       case 281: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==281);
+#line 517 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); }
+#line 6995 "sql.c"
         break;
       case 282: /* cmd ::= SHOW GRANTS FULL */
+#line 519 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_GRANTS_FULL_STMT); }
+#line 7000 "sql.c"
         break;
       case 283: /* cmd ::= SHOW GRANTS LOGS */
+#line 520 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_GRANTS_LOGS_STMT); }
+#line 7005 "sql.c"
         break;
       case 284: /* cmd ::= SHOW CLUSTER MACHINES */
+#line 521 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT); }
+#line 7010 "sql.c"
         break;
       case 285: /* cmd ::= SHOW CREATE DATABASE db_name */
+#line 522 "sql.y"
 { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy649); }
+#line 7015 "sql.c"
         break;
       case 286: /* cmd ::= SHOW CREATE TABLE full_table_name */
+#line 523 "sql.y"
 { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy600); }
+#line 7020 "sql.c"
         break;
       case 287: /* cmd ::= SHOW CREATE STABLE full_table_name */
+#line 524 "sql.y"
 { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, 
 yymsp[0].minor.yy600); }
+#line 7026 "sql.c"
         break;
       case 288: /* cmd ::= SHOW ENCRYPTIONS */
+#line 526 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ENCRYPTIONS_STMT); }
+#line 7031 "sql.c"
         break;
       case 289: /* cmd ::= SHOW QUERIES */
+#line 527 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); }
+#line 7036 "sql.c"
         break;
       case 290: /* cmd ::= SHOW SCORES */
+#line 528 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); }
+#line 7041 "sql.c"
         break;
       case 291: /* cmd ::= SHOW TOPICS */
+#line 529 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); }
+#line 7046 "sql.c"
         break;
       case 292: /* cmd ::= SHOW VARIABLES */
       case 293: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==293);
+#line 530 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); }
+#line 7052 "sql.c"
         break;
       case 294: /* cmd ::= SHOW LOCAL VARIABLES */
+#line 532 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); }
+#line 7057 "sql.c"
         break;
       case 295: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */
+#line 533 "sql.y"
 { pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy600); }
+#line 7062 "sql.c"
         break;
       case 296: /* cmd ::= SHOW BNODES */
+#line 534 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); }
+#line 7067 "sql.c"
         break;
       case 297: /* cmd ::= SHOW SNODES */
+#line 535 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); }
+#line 7072 "sql.c"
         break;
       case 298: /* cmd ::= SHOW CLUSTER */
+#line 536 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); }
+#line 7077 "sql.c"
         break;
       case 299: /* cmd ::= SHOW TRANSACTIONS */
+#line 537 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); }
+#line 7082 "sql.c"
         break;
       case 300: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */
+#line 538 "sql.y"
 { pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy600); }
+#line 7087 "sql.c"
         break;
       case 301: /* cmd ::= SHOW CONSUMERS */
+#line 539 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); }
+#line 7092 "sql.c"
         break;
       case 302: /* cmd ::= SHOW SUBSCRIPTIONS */
+#line 540 "sql.y"
 { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); }
+#line 7097 "sql.c"
         break;
       case 303: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */
+#line 541 "sql.y"
 { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); }
+#line 7102 "sql.c"
         break;
       case 304: /* cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */
+#line 542 "sql.y"
 { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy649), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649), OP_TYPE_EQUAL); }
+#line 7107 "sql.c"
         break;
       case 305: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */
+#line 543 "sql.y"
 { pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600, yymsp[-3].minor.yy748); }
+#line 7112 "sql.c"
         break;
       case 306: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */
+#line 544 "sql.y"
 { pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649), createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy649), yymsp[-4].minor.yy748); }
+#line 7117 "sql.c"
         break;
       case 307: /* cmd ::= SHOW VNODES ON DNODE NK_INTEGER */
+#line 545 "sql.y"
 { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); }
+#line 7122 "sql.c"
         break;
       case 308: /* cmd ::= SHOW VNODES */
+#line 546 "sql.y"
 { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, NULL); }
+#line 7127 "sql.c"
         break;
       case 309: /* cmd ::= SHOW db_name_cond_opt ALIVE */
+#line 548 "sql.y"
 { pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy600,    QUERY_NODE_SHOW_DB_ALIVE_STMT); }
+#line 7132 "sql.c"
         break;
       case 310: /* cmd ::= SHOW CLUSTER ALIVE */
+#line 549 "sql.y"
 { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); }
+#line 7137 "sql.c"
         break;
       case 311: /* cmd ::= SHOW db_name_cond_opt VIEWS like_pattern_opt */
+#line 550 "sql.y"
 { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VIEWS_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); }
+#line 7142 "sql.c"
         break;
       case 312: /* cmd ::= SHOW CREATE VIEW full_table_name */
+#line 551 "sql.y"
 { pCxt->pRootNode = createShowCreateViewStmt(pCxt, QUERY_NODE_SHOW_CREATE_VIEW_STMT, yymsp[0].minor.yy600); }
+#line 7147 "sql.c"
         break;
       case 313: /* cmd ::= SHOW COMPACTS */
+#line 552 "sql.y"
 { pCxt->pRootNode = createShowCompactsStmt(pCxt, QUERY_NODE_SHOW_COMPACTS_STMT); }
+#line 7152 "sql.c"
         break;
       case 314: /* cmd ::= SHOW COMPACT NK_INTEGER */
+#line 553 "sql.y"
 { pCxt->pRootNode = createShowCompactDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); }
+#line 7157 "sql.c"
         break;
       case 315: /* table_kind_db_name_cond_opt ::= */
+#line 557 "sql.y"
 { yymsp[1].minor.yy125.kind = SHOW_KIND_ALL; yymsp[1].minor.yy125.dbName = nil_token; }
+#line 7162 "sql.c"
         break;
       case 316: /* table_kind_db_name_cond_opt ::= table_kind */
+#line 558 "sql.y"
 { yylhsminor.yy125.kind = yymsp[0].minor.yy245; yylhsminor.yy125.dbName = nil_token; }
+#line 7167 "sql.c"
   yymsp[0].minor.yy125 = yylhsminor.yy125;
         break;
       case 317: /* table_kind_db_name_cond_opt ::= db_name NK_DOT */
+#line 559 "sql.y"
 { yylhsminor.yy125.kind = SHOW_KIND_ALL; yylhsminor.yy125.dbName = yymsp[-1].minor.yy649; }
+#line 7173 "sql.c"
   yymsp[-1].minor.yy125 = yylhsminor.yy125;
         break;
       case 318: /* table_kind_db_name_cond_opt ::= table_kind db_name NK_DOT */
+#line 560 "sql.y"
 { yylhsminor.yy125.kind = yymsp[-2].minor.yy245; yylhsminor.yy125.dbName = yymsp[-1].minor.yy649; }
+#line 7179 "sql.c"
   yymsp[-2].minor.yy125 = yylhsminor.yy125;
         break;
       case 319: /* table_kind ::= NORMAL */
+#line 564 "sql.y"
 { yymsp[0].minor.yy245 = SHOW_KIND_TABLES_NORMAL; }
+#line 7185 "sql.c"
         break;
       case 320: /* table_kind ::= CHILD */
+#line 565 "sql.y"
 { yymsp[0].minor.yy245 = SHOW_KIND_TABLES_CHILD; }
+#line 7190 "sql.c"
         break;
       case 321: /* db_name_cond_opt ::= */
       case 326: /* from_db_opt ::= */ yytestcase(yyruleno==326);
+#line 567 "sql.y"
 { yymsp[1].minor.yy600 = createDefaultDatabaseCondValue(pCxt); }
+#line 7196 "sql.c"
         break;
       case 322: /* db_name_cond_opt ::= db_name NK_DOT */
+#line 568 "sql.y"
 { yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy649); }
+#line 7201 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 324: /* like_pattern_opt ::= LIKE NK_STRING */
+#line 571 "sql.y"
 { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); }
+#line 7207 "sql.c"
         break;
       case 325: /* table_name_cond ::= table_name */
+#line 573 "sql.y"
 { yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649); }
+#line 7212 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 327: /* from_db_opt ::= FROM db_name */
+#line 576 "sql.y"
 { yymsp[-1].minor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649); }
+#line 7218 "sql.c"
         break;
       case 331: /* tag_item ::= TBNAME */
+#line 584 "sql.y"
 { yylhsminor.yy600 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); }
+#line 7223 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 334: /* tag_item ::= column_name column_alias */
+#line 587 "sql.y"
 { yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy649), &yymsp[0].minor.yy649); }
+#line 7229 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 335: /* tag_item ::= column_name AS column_alias */
+#line 588 "sql.y"
 { yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy649), &yymsp[0].minor.yy649); }
+#line 7235 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 336: /* db_kind_opt ::= */
+#line 592 "sql.y"
 { yymsp[1].minor.yy245 = SHOW_KIND_ALL; }
+#line 7241 "sql.c"
         break;
       case 337: /* db_kind_opt ::= USER */
+#line 593 "sql.y"
 { yymsp[0].minor.yy245 = SHOW_KIND_DATABASES_USER; }
+#line 7246 "sql.c"
         break;
       case 338: /* db_kind_opt ::= SYSTEM */
+#line 594 "sql.y"
 { yymsp[0].minor.yy245 = SHOW_KIND_DATABASES_SYSTEM; }
+#line 7251 "sql.c"
         break;
       case 339: /* cmd ::= CREATE TSMA not_exists_opt tsma_name ON full_table_name tsma_func_list INTERVAL NK_LP duration_literal NK_RP */
+#line 600 "sql.y"
 { pCxt->pRootNode = createCreateTSMAStmt(pCxt, yymsp[-8].minor.yy705, &yymsp[-7].minor.yy649, yymsp[-4].minor.yy600, yymsp[-5].minor.yy600, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 7256 "sql.c"
         break;
       case 340: /* cmd ::= CREATE RECURSIVE TSMA not_exists_opt tsma_name ON full_table_name INTERVAL NK_LP duration_literal NK_RP */
+#line 602 "sql.y"
 { pCxt->pRootNode = createCreateTSMAStmt(pCxt, yymsp[-7].minor.yy705, &yymsp[-6].minor.yy649, NULL, yymsp[-4].minor.yy600, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 7261 "sql.c"
         break;
       case 341: /* cmd ::= DROP TSMA exists_opt full_tsma_name */
+#line 603 "sql.y"
 { pCxt->pRootNode = createDropTSMAStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); }
+#line 7266 "sql.c"
         break;
       case 342: /* cmd ::= SHOW db_name_cond_opt TSMAS */
+#line 604 "sql.y"
 { pCxt->pRootNode = createShowTSMASStmt(pCxt, yymsp[-1].minor.yy600); }
+#line 7271 "sql.c"
         break;
       case 345: /* tsma_func_list ::= FUNCTION NK_LP func_list NK_RP */
+#line 611 "sql.y"
 { yymsp[-3].minor.yy600 = createTSMAOptions(pCxt, yymsp[-1].minor.yy748); }
+#line 7276 "sql.c"
         break;
       case 346: /* cmd ::= CREATE SMA INDEX not_exists_opt col_name ON full_table_name index_options */
+#line 615 "sql.y"
 { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy705, yymsp[-3].minor.yy600, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); }
+#line 7281 "sql.c"
         break;
       case 347: /* cmd ::= CREATE INDEX not_exists_opt col_name ON full_table_name NK_LP col_name_list NK_RP */
+#line 617 "sql.y"
 { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy705, yymsp[-5].minor.yy600, yymsp[-3].minor.yy600, yymsp[-1].minor.yy748, NULL); }
+#line 7286 "sql.c"
         break;
       case 348: /* cmd ::= DROP INDEX exists_opt full_index_name */
+#line 618 "sql.y"
 { pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); }
+#line 7291 "sql.c"
         break;
       case 349: /* full_index_name ::= index_name */
+#line 620 "sql.y"
 { yylhsminor.yy600 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy649); }
+#line 7296 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 350: /* full_index_name ::= db_name NK_DOT index_name */
+#line 621 "sql.y"
 { yylhsminor.yy600 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649); }
+#line 7302 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 351: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */
+#line 624 "sql.y"
 { yymsp[-9].minor.yy600 = createIndexOption(pCxt, yymsp[-7].minor.yy748, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 7308 "sql.c"
         break;
       case 352: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */
+#line 627 "sql.y"
 { yymsp[-11].minor.yy600 = createIndexOption(pCxt, yymsp[-9].minor.yy748, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 7313 "sql.c"
         break;
       case 355: /* func ::= sma_func_name NK_LP expression_list NK_RP */
+#line 634 "sql.y"
 { yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[-3].minor.yy649, yymsp[-1].minor.yy748); }
+#line 7318 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 356: /* sma_func_name ::= function_name */
       case 630: /* alias_opt ::= table_alias */ yytestcase(yyruleno==630);
+#line 638 "sql.y"
 { yylhsminor.yy649 = yymsp[0].minor.yy649; }
+#line 7325 "sql.c"
   yymsp[0].minor.yy649 = yylhsminor.yy649;
         break;
       case 361: /* sma_stream_opt ::= */
       case 411: /* stream_options ::= */ yytestcase(yyruleno==411);
+#line 644 "sql.y"
 { yymsp[1].minor.yy600 = createStreamOptions(pCxt); }
+#line 7332 "sql.c"
         break;
       case 362: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */
+#line 645 "sql.y"
 { ((SStreamOptions*)yymsp[-2].minor.yy600)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; }
+#line 7337 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 363: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */
+#line 646 "sql.y"
 { ((SStreamOptions*)yymsp[-2].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; }
+#line 7343 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 364: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */
+#line 647 "sql.y"
 { ((SStreamOptions*)yymsp[-2].minor.yy600)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; }
+#line 7349 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 365: /* with_meta ::= AS */
+#line 652 "sql.y"
 { yymsp[0].minor.yy756 = 0; }
+#line 7355 "sql.c"
         break;
       case 366: /* with_meta ::= WITH META AS */
+#line 653 "sql.y"
 { yymsp[-2].minor.yy756 = 1; }
+#line 7360 "sql.c"
         break;
       case 367: /* with_meta ::= ONLY META AS */
+#line 654 "sql.y"
 { yymsp[-2].minor.yy756 = 2; }
+#line 7365 "sql.c"
         break;
       case 368: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */
+#line 656 "sql.y"
 { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy705, &yymsp[-2].minor.yy649, yymsp[0].minor.yy600); }
+#line 7370 "sql.c"
         break;
       case 369: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */
+#line 658 "sql.y"
 { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy705, &yymsp[-3].minor.yy649, &yymsp[0].minor.yy649, yymsp[-2].minor.yy756); }
+#line 7375 "sql.c"
         break;
       case 370: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */
+#line 660 "sql.y"
 { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy705, &yymsp[-4].minor.yy649, yymsp[-1].minor.yy600, yymsp[-3].minor.yy756, yymsp[0].minor.yy600); }
+#line 7380 "sql.c"
         break;
       case 371: /* cmd ::= DROP TOPIC exists_opt topic_name */
+#line 662 "sql.y"
 { pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); }
+#line 7385 "sql.c"
         break;
       case 372: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */
+#line 663 "sql.y"
 { pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy705, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649); }
+#line 7390 "sql.c"
         break;
       case 373: /* cmd ::= DESC full_table_name */
       case 374: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==374);
+#line 666 "sql.y"
 { pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy600); }
+#line 7396 "sql.c"
         break;
       case 375: /* cmd ::= RESET QUERY CACHE */
+#line 670 "sql.y"
 { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); }
+#line 7401 "sql.c"
         break;
       case 376: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */
       case 377: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==377);
+#line 673 "sql.y"
 { pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy705, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 7407 "sql.c"
         break;
       case 380: /* explain_options ::= */
+#line 681 "sql.y"
 { yymsp[1].minor.yy600 = createDefaultExplainOptions(pCxt); }
+#line 7412 "sql.c"
         break;
       case 381: /* explain_options ::= explain_options VERBOSE NK_BOOL */
+#line 682 "sql.y"
 { yylhsminor.yy600 = setExplainVerbose(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); }
+#line 7417 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 382: /* explain_options ::= explain_options RATIO NK_FLOAT */
+#line 683 "sql.y"
 { yylhsminor.yy600 = setExplainRatio(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); }
+#line 7423 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 383: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */
+#line 688 "sql.y"
 { pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy705, yymsp[-9].minor.yy705, &yymsp[-6].minor.yy649, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy400, yymsp[-1].minor.yy756, &yymsp[0].minor.yy649, yymsp[-10].minor.yy705); }
+#line 7429 "sql.c"
         break;
       case 384: /* cmd ::= DROP FUNCTION exists_opt function_name */
+#line 689 "sql.y"
 { pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); }
+#line 7434 "sql.c"
         break;
       case 389: /* language_opt ::= */
       case 434: /* on_vgroup_id ::= */ yytestcase(yyruleno==434);
+#line 703 "sql.y"
 { yymsp[1].minor.yy649 = nil_token; }
+#line 7440 "sql.c"
         break;
       case 390: /* language_opt ::= LANGUAGE NK_STRING */
       case 435: /* on_vgroup_id ::= ON NK_INTEGER */ yytestcase(yyruleno==435);
+#line 704 "sql.y"
 { yymsp[-1].minor.yy649 = yymsp[0].minor.yy0; }
+#line 7446 "sql.c"
         break;
       case 393: /* cmd ::= CREATE or_replace_opt VIEW full_view_name AS query_or_subquery */
+#line 713 "sql.y"
 { pCxt->pRootNode = createCreateViewStmt(pCxt, yymsp[-4].minor.yy705, yymsp[-2].minor.yy600, &yymsp[-1].minor.yy0, yymsp[0].minor.yy600); }
+#line 7451 "sql.c"
         break;
       case 394: /* cmd ::= DROP VIEW exists_opt full_view_name */
+#line 714 "sql.y"
 { pCxt->pRootNode = createDropViewStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); }
+#line 7456 "sql.c"
         break;
       case 395: /* full_view_name ::= view_name */
+#line 716 "sql.y"
 { yylhsminor.yy600 = createViewNode(pCxt, NULL, &yymsp[0].minor.yy649); }
+#line 7461 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 396: /* full_view_name ::= db_name NK_DOT view_name */
+#line 717 "sql.y"
 { yylhsminor.yy600 = createViewNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649); }
+#line 7467 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 397: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */
+#line 722 "sql.y"
 { pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy705, &yymsp[-8].minor.yy649, yymsp[-5].minor.yy600, yymsp[-7].minor.yy600, yymsp[-3].minor.yy748, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, yymsp[-4].minor.yy748); }
+#line 7473 "sql.c"
         break;
       case 398: /* cmd ::= DROP STREAM exists_opt stream_name */
+#line 723 "sql.y"
 { pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); }
+#line 7478 "sql.c"
         break;
       case 399: /* cmd ::= PAUSE STREAM exists_opt stream_name */
+#line 724 "sql.y"
 { pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); }
+#line 7483 "sql.c"
         break;
       case 400: /* cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */
+#line 725 "sql.y"
 { pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy705, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); }
+#line 7488 "sql.c"
         break;
       case 405: /* column_stream_def ::= column_name stream_col_options */
+#line 738 "sql.y"
 { yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy649, createDataType(TSDB_DATA_TYPE_NULL), yymsp[0].minor.yy600); }
+#line 7493 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 406: /* stream_col_options ::= */
       case 743: /* column_options ::= */ yytestcase(yyruleno==743);
+#line 739 "sql.y"
 { yymsp[1].minor.yy600 = createDefaultColumnOptions(pCxt); }
+#line 7500 "sql.c"
         break;
       case 407: /* stream_col_options ::= stream_col_options PRIMARY KEY */
       case 744: /* column_options ::= column_options PRIMARY KEY */ yytestcase(yyruleno==744);
+#line 740 "sql.y"
 { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_PRIMARYKEY, NULL); }
+#line 7506 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 412: /* stream_options ::= stream_options TRIGGER AT_ONCE */
       case 413: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==413);
+#line 750 "sql.y"
 { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); }
+#line 7513 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 414: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */
+#line 752 "sql.y"
 { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-3].minor.yy600, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); }
+#line 7519 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 415: /* stream_options ::= stream_options WATERMARK duration_literal */
+#line 753 "sql.y"
 { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); }
+#line 7525 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 416: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */
+#line 754 "sql.y"
 { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-3].minor.yy600, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); }
+#line 7531 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 417: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */
+#line 755 "sql.y"
 { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); }
+#line 7537 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 418: /* stream_options ::= stream_options DELETE_MARK duration_literal */
+#line 756 "sql.y"
 { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); }
+#line 7543 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 419: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */
+#line 757 "sql.y"
 { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-3].minor.yy600, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); }
+#line 7549 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 421: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */
       case 687: /* sliding_opt ::= SLIDING NK_LP interval_sliding_duration_literal NK_RP */ yytestcase(yyruleno==687);
       case 711: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==711);
+#line 760 "sql.y"
 { yymsp[-3].minor.yy600 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy600); }
+#line 7557 "sql.c"
         break;
       case 424: /* cmd ::= KILL CONNECTION NK_INTEGER */
+#line 768 "sql.y"
 { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); }
+#line 7562 "sql.c"
         break;
       case 425: /* cmd ::= KILL QUERY NK_STRING */
+#line 769 "sql.y"
 { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); }
+#line 7567 "sql.c"
         break;
       case 426: /* cmd ::= KILL TRANSACTION NK_INTEGER */
+#line 770 "sql.y"
 { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); }
+#line 7572 "sql.c"
         break;
       case 427: /* cmd ::= KILL COMPACT NK_INTEGER */
+#line 771 "sql.y"
 { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_COMPACT_STMT, &yymsp[0].minor.yy0); }
+#line 7577 "sql.c"
         break;
       case 428: /* cmd ::= BALANCE VGROUP */
+#line 774 "sql.y"
 { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); }
+#line 7582 "sql.c"
         break;
       case 429: /* cmd ::= BALANCE VGROUP LEADER on_vgroup_id */
+#line 775 "sql.y"
 { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &yymsp[0].minor.yy649); }
+#line 7587 "sql.c"
         break;
       case 430: /* cmd ::= BALANCE VGROUP LEADER DATABASE db_name */
+#line 776 "sql.y"
 { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &yymsp[0].minor.yy649); }
+#line 7592 "sql.c"
         break;
       case 431: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */
+#line 777 "sql.y"
 { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); }
+#line 7597 "sql.c"
         break;
       case 432: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */
+#line 778 "sql.y"
 { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy748); }
+#line 7602 "sql.c"
         break;
       case 433: /* cmd ::= SPLIT VGROUP NK_INTEGER */
+#line 779 "sql.y"
 { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); }
+#line 7607 "sql.c"
         break;
       case 436: /* dnode_list ::= DNODE NK_INTEGER */
+#line 788 "sql.y"
 { yymsp[-1].minor.yy748 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); }
+#line 7612 "sql.c"
         break;
       case 438: /* cmd ::= DELETE FROM full_table_name where_clause_opt */
+#line 795 "sql.y"
 { pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 7617 "sql.c"
         break;
       case 441: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */
+#line 804 "sql.y"
 { yymsp[-6].minor.yy600 = createInsertStmt(pCxt, yymsp[-4].minor.yy600, yymsp[-2].minor.yy748, yymsp[0].minor.yy600); }
+#line 7622 "sql.c"
         break;
       case 442: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */
+#line 805 "sql.y"
 { yymsp[-3].minor.yy600 = createInsertStmt(pCxt, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); }
+#line 7627 "sql.c"
         break;
       case 443: /* tags_literal ::= NK_INTEGER */
       case 455: /* tags_literal ::= NK_BIN */ yytestcase(yyruleno==455);
       case 464: /* tags_literal ::= NK_HEX */ yytestcase(yyruleno==464);
+#line 808 "sql.y"
 { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0, NULL); }
+#line 7634 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 444: /* tags_literal ::= NK_INTEGER NK_PLUS duration_literal */
@@ -6903,12 +7642,14 @@ yymsp[0].minor.yy600); }
       case 466: /* tags_literal ::= NK_HEX NK_MINUS duration_literal */ yytestcase(yyruleno==466);
       case 474: /* tags_literal ::= NK_STRING NK_PLUS duration_literal */ yytestcase(yyruleno==474);
       case 475: /* tags_literal ::= NK_STRING NK_MINUS duration_literal */ yytestcase(yyruleno==475);
+#line 809 "sql.y"
 {
                                                                                     SToken l = yymsp[-2].minor.yy0;
                                                                                     SToken r = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     l.n = (r.z + r.n) - l.z;  
                                                                                     yylhsminor.yy600 = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, yymsp[0].minor.yy600);
                                                                                   }
+#line 7652 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 446: /* tags_literal ::= NK_PLUS NK_INTEGER */
@@ -6917,11 +7658,13 @@ yymsp[0].minor.yy600); }
       case 461: /* tags_literal ::= NK_MINUS NK_BIN */ yytestcase(yyruleno==461);
       case 467: /* tags_literal ::= NK_PLUS NK_HEX */ yytestcase(yyruleno==467);
       case 470: /* tags_literal ::= NK_MINUS NK_HEX */ yytestcase(yyruleno==470);
+#line 821 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &t, NULL);
                                                                                   }
+#line 7667 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 447: /* tags_literal ::= NK_PLUS NK_INTEGER NK_PLUS duration_literal */
@@ -6936,71 +7679,97 @@ yymsp[0].minor.yy600); }
       case 469: /* tags_literal ::= NK_PLUS NK_HEX NK_MINUS duration_literal */ yytestcase(yyruleno==469);
       case 471: /* tags_literal ::= NK_MINUS NK_HEX NK_PLUS duration_literal */ yytestcase(yyruleno==471);
       case 472: /* tags_literal ::= NK_MINUS NK_HEX NK_MINUS duration_literal */ yytestcase(yyruleno==472);
+#line 826 "sql.y"
 {
                                                                                     SToken l = yymsp[-3].minor.yy0;
                                                                                     SToken r = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     l.n = (r.z + r.n) - l.z;  
                                                                                     yylhsminor.yy600 = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, yymsp[0].minor.yy600);
                                                                                   }
+#line 7689 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 452: /* tags_literal ::= NK_FLOAT */
+#line 855 "sql.y"
 { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0, NULL); }
+#line 7695 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 453: /* tags_literal ::= NK_PLUS NK_FLOAT */
       case 454: /* tags_literal ::= NK_MINUS NK_FLOAT */ yytestcase(yyruleno==454);
+#line 856 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t, NULL);
                                                                                   }
+#line 7706 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 473: /* tags_literal ::= NK_STRING */
+#line 962 "sql.y"
 { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0, NULL); }
+#line 7712 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 476: /* tags_literal ::= NK_BOOL */
+#line 975 "sql.y"
 { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0, NULL); }
+#line 7718 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 477: /* tags_literal ::= NULL */
+#line 976 "sql.y"
 { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0, NULL); }
+#line 7724 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 478: /* tags_literal ::= literal_func */
+#line 978 "sql.y"
 { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_BINARY, NULL, yymsp[0].minor.yy600); }
+#line 7730 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 479: /* tags_literal ::= literal_func NK_PLUS duration_literal */
       case 480: /* tags_literal ::= literal_func NK_MINUS duration_literal */ yytestcase(yyruleno==480);
+#line 979 "sql.y"
 {
                                                                                     SToken l = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken r = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     l.n = (r.z + r.n) - l.z;
                                                                                     yylhsminor.yy600 = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, yymsp[-2].minor.yy600, yymsp[0].minor.yy600);
                                                                                   }
+#line 7742 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 483: /* literal ::= NK_INTEGER */
+#line 998 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); }
+#line 7748 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 484: /* literal ::= NK_FLOAT */
+#line 999 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); }
+#line 7754 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 485: /* literal ::= NK_STRING */
+#line 1000 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); }
+#line 7760 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 486: /* literal ::= NK_BOOL */
+#line 1001 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); }
+#line 7766 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 487: /* literal ::= TIMESTAMP NK_STRING */
+#line 1002 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); }
+#line 7772 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 488: /* literal ::= duration_literal */
@@ -7023,64 +7792,90 @@ yymsp[0].minor.yy600); }
       case 714: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==714);
       case 717: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==717);
       case 719: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==719);
+#line 1003 "sql.y"
 { yylhsminor.yy600 = yymsp[0].minor.yy600; }
+#line 7797 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 489: /* literal ::= NULL */
+#line 1004 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); }
+#line 7803 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 490: /* literal ::= NK_QUESTION */
+#line 1005 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); }
+#line 7809 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 491: /* duration_literal ::= NK_VARIABLE */
       case 688: /* interval_sliding_duration_literal ::= NK_VARIABLE */ yytestcase(yyruleno==688);
       case 689: /* interval_sliding_duration_literal ::= NK_STRING */ yytestcase(yyruleno==689);
       case 690: /* interval_sliding_duration_literal ::= NK_INTEGER */ yytestcase(yyruleno==690);
+#line 1007 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); }
+#line 7818 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 492: /* signed ::= NK_INTEGER */
+#line 1009 "sql.y"
 { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); }
+#line 7824 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 493: /* signed ::= NK_PLUS NK_INTEGER */
+#line 1010 "sql.y"
 { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); }
+#line 7830 "sql.c"
         break;
       case 494: /* signed ::= NK_MINUS NK_INTEGER */
+#line 1011 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t);
                                                                                   }
+#line 7839 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 495: /* signed ::= NK_FLOAT */
+#line 1016 "sql.y"
 { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); }
+#line 7845 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 496: /* signed ::= NK_PLUS NK_FLOAT */
+#line 1017 "sql.y"
 { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); }
+#line 7851 "sql.c"
         break;
       case 497: /* signed ::= NK_MINUS NK_FLOAT */
+#line 1018 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t);
                                                                                   }
+#line 7860 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 499: /* signed_literal ::= NK_STRING */
+#line 1025 "sql.y"
 { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); }
+#line 7866 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 500: /* signed_literal ::= NK_BOOL */
+#line 1026 "sql.y"
 { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); }
+#line 7872 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 501: /* signed_literal ::= TIMESTAMP NK_STRING */
+#line 1027 "sql.y"
 { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); }
+#line 7878 "sql.c"
         break;
       case 502: /* signed_literal ::= duration_literal */
       case 504: /* signed_literal ::= literal_func */ yytestcase(yyruleno==504);
@@ -7090,118 +7885,156 @@ yymsp[0].minor.yy600); }
       case 718: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==718);
       case 720: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==720);
       case 733: /* search_condition ::= common_expression */ yytestcase(yyruleno==733);
+#line 1028 "sql.y"
 { yylhsminor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); }
+#line 7890 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 503: /* signed_literal ::= NULL */
+#line 1029 "sql.y"
 { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); }
+#line 7896 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 505: /* signed_literal ::= NK_QUESTION */
+#line 1031 "sql.y"
 { yylhsminor.yy600 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); }
+#line 7902 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 524: /* expression ::= pseudo_column */
+#line 1097 "sql.y"
 { yylhsminor.yy600 = yymsp[0].minor.yy600; setRawExprNodeIsPseudoColumn(pCxt, yylhsminor.yy600, true); }
+#line 7908 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 528: /* expression ::= NK_LP expression NK_RP */
       case 616: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==616);
       case 732: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==732);
+#line 1101 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 7916 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 529: /* expression ::= NK_PLUS expr_or_subquery */
+#line 1102 "sql.y"
 {
                                                                                     SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy600));
                                                                                   }
+#line 7925 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 530: /* expression ::= NK_MINUS expr_or_subquery */
+#line 1106 "sql.y"
 {
                                                                                     SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL));
                                                                                   }
+#line 7934 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 531: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */
+#line 1110 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 7944 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 532: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */
+#line 1115 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 7954 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 533: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */
+#line 1120 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 7964 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 534: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */
+#line 1125 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 7974 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 535: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */
+#line 1130 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 7984 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 536: /* expression ::= column_reference NK_ARROW NK_STRING */
+#line 1135 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)));
                                                                                   }
+#line 7993 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 537: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */
+#line 1139 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 8003 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 538: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */
+#line 1144 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 8013 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 541: /* column_reference ::= column_name */
+#line 1155 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy649, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy649)); }
+#line 8019 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 542: /* column_reference ::= table_name NK_DOT column_name */
+#line 1156 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649, createColumnNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649)); }
+#line 8025 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 543: /* column_reference ::= NK_ALIAS */
+#line 1157 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); }
+#line 8031 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 544: /* column_reference ::= table_name NK_DOT NK_ALIAS */
+#line 1158 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0, createColumnNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0)); }
+#line 8037 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 545: /* pseudo_column ::= ROWTS */
@@ -7217,243 +8050,356 @@ yymsp[0].minor.yy600); }
       case 556: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==556);
       case 563: /* literal_func ::= NOW */ yytestcase(yyruleno==563);
       case 564: /* literal_func ::= TODAY */ yytestcase(yyruleno==564);
+#line 1160 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); }
+#line 8055 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 547: /* pseudo_column ::= table_name NK_DOT TBNAME */
+#line 1162 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy649)))); }
+#line 8061 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 557: /* function_expression ::= function_name NK_LP expression_list NK_RP */
       case 558: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==558);
+#line 1173 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy649, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy649, yymsp[-1].minor.yy748)); }
+#line 8068 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 559: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */
       case 560: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name_default_len NK_RP */ yytestcase(yyruleno==560);
+#line 1176 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy400)); }
+#line 8075 "sql.c"
   yymsp[-5].minor.yy600 = yylhsminor.yy600;
         break;
       case 562: /* literal_func ::= noarg_func NK_LP NK_RP */
+#line 1182 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy649, NULL)); }
+#line 8081 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 578: /* star_func_para_list ::= NK_STAR */
+#line 1207 "sql.y"
 { yylhsminor.yy748 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); }
+#line 8087 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 583: /* star_func_para ::= table_name NK_DOT NK_STAR */
       case 668: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==668);
+#line 1216 "sql.y"
 { yylhsminor.yy600 = createColumnNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0); }
+#line 8094 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 584: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */
+#line 1219 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy748, yymsp[-1].minor.yy600)); }
+#line 8100 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 585: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */
+#line 1221 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-2].minor.yy748, yymsp[-1].minor.yy600)); }
+#line 8106 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 588: /* when_then_expr ::= WHEN common_expression THEN common_expression */
+#line 1228 "sql.y"
 { yymsp[-3].minor.yy600 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); }
+#line 8112 "sql.c"
         break;
       case 590: /* case_when_else_opt ::= ELSE common_expression */
+#line 1231 "sql.y"
 { yymsp[-1].minor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); }
+#line 8117 "sql.c"
         break;
       case 591: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */
       case 596: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==596);
+#line 1234 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy292, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 8127 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 592: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */
+#line 1241 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 8137 "sql.c"
   yymsp[-4].minor.yy600 = yylhsminor.yy600;
         break;
       case 593: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */
+#line 1247 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 8147 "sql.c"
   yymsp[-5].minor.yy600 = yylhsminor.yy600;
         break;
       case 594: /* predicate ::= expr_or_subquery IS NULL */
+#line 1252 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), NULL));
                                                                                   }
+#line 8156 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 595: /* predicate ::= expr_or_subquery IS NOT NULL */
+#line 1256 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL));
                                                                                   }
+#line 8165 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 597: /* compare_op ::= NK_LT */
+#line 1268 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_LOWER_THAN; }
+#line 8171 "sql.c"
         break;
       case 598: /* compare_op ::= NK_GT */
+#line 1269 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_GREATER_THAN; }
+#line 8176 "sql.c"
         break;
       case 599: /* compare_op ::= NK_LE */
+#line 1270 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_LOWER_EQUAL; }
+#line 8181 "sql.c"
         break;
       case 600: /* compare_op ::= NK_GE */
+#line 1271 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_GREATER_EQUAL; }
+#line 8186 "sql.c"
         break;
       case 601: /* compare_op ::= NK_NE */
+#line 1272 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_NOT_EQUAL; }
+#line 8191 "sql.c"
         break;
       case 602: /* compare_op ::= NK_EQ */
+#line 1273 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_EQUAL; }
+#line 8196 "sql.c"
         break;
       case 603: /* compare_op ::= LIKE */
+#line 1274 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_LIKE; }
+#line 8201 "sql.c"
         break;
       case 604: /* compare_op ::= NOT LIKE */
+#line 1275 "sql.y"
 { yymsp[-1].minor.yy292 = OP_TYPE_NOT_LIKE; }
+#line 8206 "sql.c"
         break;
       case 605: /* compare_op ::= MATCH */
+#line 1276 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_MATCH; }
+#line 8211 "sql.c"
         break;
       case 606: /* compare_op ::= NMATCH */
+#line 1277 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_NMATCH; }
+#line 8216 "sql.c"
         break;
       case 607: /* compare_op ::= CONTAINS */
+#line 1278 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_JSON_CONTAINS; }
+#line 8221 "sql.c"
         break;
       case 608: /* in_op ::= IN */
+#line 1282 "sql.y"
 { yymsp[0].minor.yy292 = OP_TYPE_IN; }
+#line 8226 "sql.c"
         break;
       case 609: /* in_op ::= NOT IN */
+#line 1283 "sql.y"
 { yymsp[-1].minor.yy292 = OP_TYPE_NOT_IN; }
+#line 8231 "sql.c"
         break;
       case 610: /* in_predicate_value ::= NK_LP literal_list NK_RP */
+#line 1285 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy748)); }
+#line 8236 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 612: /* boolean_value_expression ::= NOT boolean_primary */
+#line 1289 "sql.y"
 {
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL));
                                                                                   }
+#line 8245 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 613: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */
+#line 1294 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 8255 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 614: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */
+#line 1300 "sql.y"
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600);
                                                                                     yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)));
                                                                                   }
+#line 8265 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 622: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */
+#line 1318 "sql.y"
 { yylhsminor.yy600 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, JOIN_STYPE_NONE, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, NULL); }
+#line 8271 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 625: /* table_primary ::= table_name alias_opt */
+#line 1324 "sql.y"
 { yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); }
+#line 8277 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 626: /* table_primary ::= db_name NK_DOT table_name alias_opt */
+#line 1325 "sql.y"
 { yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-3].minor.yy649, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); }
+#line 8283 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 627: /* table_primary ::= subquery alias_opt */
+#line 1326 "sql.y"
 { yylhsminor.yy600 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy649); }
+#line 8289 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 629: /* alias_opt ::= */
+#line 1331 "sql.y"
 { yymsp[1].minor.yy649 = nil_token;  }
+#line 8295 "sql.c"
         break;
       case 631: /* alias_opt ::= AS table_alias */
+#line 1333 "sql.y"
 { yymsp[-1].minor.yy649 = yymsp[0].minor.yy649; }
+#line 8300 "sql.c"
         break;
       case 632: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */
       case 633: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==633);
+#line 1335 "sql.y"
 { yymsp[-2].minor.yy600 = yymsp[-1].minor.yy600; }
+#line 8306 "sql.c"
         break;
       case 634: /* joined_table ::= table_reference join_type join_subtype JOIN table_reference join_on_clause_opt window_offset_clause_opt jlimit_clause_opt */
+#line 1341 "sql.y"
 { 
                                                                                     yylhsminor.yy600 = createJoinTableNode(pCxt, yymsp[-6].minor.yy564, yymsp[-5].minor.yy758, yymsp[-7].minor.yy600, yymsp[-3].minor.yy600, yymsp[-2].minor.yy600); 
                                                                                     yylhsminor.yy600 = addWindowOffsetClause(pCxt, yylhsminor.yy600, yymsp[-1].minor.yy600);
                                                                                     yylhsminor.yy600 = addJLimitClause(pCxt, yylhsminor.yy600, yymsp[0].minor.yy600);
                                                                                   }
+#line 8315 "sql.c"
   yymsp[-7].minor.yy600 = yylhsminor.yy600;
         break;
       case 635: /* join_type ::= */
+#line 1349 "sql.y"
 { yymsp[1].minor.yy564 = JOIN_TYPE_INNER; }
+#line 8321 "sql.c"
         break;
       case 636: /* join_type ::= INNER */
+#line 1350 "sql.y"
 { yymsp[0].minor.yy564 = JOIN_TYPE_INNER; }
+#line 8326 "sql.c"
         break;
       case 637: /* join_type ::= LEFT */
+#line 1351 "sql.y"
 { yymsp[0].minor.yy564 = JOIN_TYPE_LEFT; }
+#line 8331 "sql.c"
         break;
       case 638: /* join_type ::= RIGHT */
+#line 1352 "sql.y"
 { yymsp[0].minor.yy564 = JOIN_TYPE_RIGHT; }
+#line 8336 "sql.c"
         break;
       case 639: /* join_type ::= FULL */
+#line 1353 "sql.y"
 { yymsp[0].minor.yy564 = JOIN_TYPE_FULL; }
+#line 8341 "sql.c"
         break;
       case 640: /* join_subtype ::= */
+#line 1357 "sql.y"
 { yymsp[1].minor.yy758 = JOIN_STYPE_NONE; }
+#line 8346 "sql.c"
         break;
       case 641: /* join_subtype ::= OUTER */
+#line 1358 "sql.y"
 { yymsp[0].minor.yy758 = JOIN_STYPE_OUTER; }
+#line 8351 "sql.c"
         break;
       case 642: /* join_subtype ::= SEMI */
+#line 1359 "sql.y"
 { yymsp[0].minor.yy758 = JOIN_STYPE_SEMI; }
+#line 8356 "sql.c"
         break;
       case 643: /* join_subtype ::= ANTI */
+#line 1360 "sql.y"
 { yymsp[0].minor.yy758 = JOIN_STYPE_ANTI; }
+#line 8361 "sql.c"
         break;
       case 644: /* join_subtype ::= ASOF */
+#line 1361 "sql.y"
 { yymsp[0].minor.yy758 = JOIN_STYPE_ASOF; }
+#line 8366 "sql.c"
         break;
       case 645: /* join_subtype ::= WINDOW */
+#line 1362 "sql.y"
 { yymsp[0].minor.yy758 = JOIN_STYPE_WIN; }
+#line 8371 "sql.c"
         break;
       case 649: /* window_offset_clause_opt ::= WINDOW_OFFSET NK_LP window_offset_literal NK_COMMA window_offset_literal NK_RP */
+#line 1369 "sql.y"
 { yymsp[-5].minor.yy600 = createWindowOffsetNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 8376 "sql.c"
         break;
       case 650: /* window_offset_literal ::= NK_VARIABLE */
+#line 1371 "sql.y"
 { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createTimeOffsetValueNode(pCxt, &yymsp[0].minor.yy0)); }
+#line 8381 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 651: /* window_offset_literal ::= NK_MINUS NK_VARIABLE */
+#line 1372 "sql.y"
 {
                                                                                     SToken t = yymsp[-1].minor.yy0;
                                                                                     t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z;
                                                                                     yylhsminor.yy600 = createRawExprNode(pCxt, &t, createTimeOffsetValueNode(pCxt, &t)); 
                                                                                   }
+#line 8391 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 653: /* jlimit_clause_opt ::= JLIMIT NK_INTEGER */
       case 724: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ yytestcase(yyruleno==724);
       case 728: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==728);
+#line 1379 "sql.y"
 { yymsp[-1].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); }
+#line 8399 "sql.c"
         break;
       case 654: /* query_specification ::= SELECT hint_list set_quantifier_opt tag_mode_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */
+#line 1385 "sql.y"
 {
                                                                                     yymsp[-13].minor.yy600 = createSelectStmt(pCxt, yymsp[-11].minor.yy705, yymsp[-9].minor.yy748, yymsp[-8].minor.yy600, yymsp[-12].minor.yy748);
                                                                                     yymsp[-13].minor.yy600 = setSelectStmtTagMode(pCxt, yymsp[-13].minor.yy600, yymsp[-10].minor.yy705);
@@ -7466,158 +8412,245 @@ yymsp[0].minor.yy600); }
                                                                                     yymsp[-13].minor.yy600 = addEveryClause(pCxt, yymsp[-13].minor.yy600, yymsp[-4].minor.yy600);
                                                                                     yymsp[-13].minor.yy600 = addFillClause(pCxt, yymsp[-13].minor.yy600, yymsp[-3].minor.yy600);
                                                                                   }
+#line 8415 "sql.c"
         break;
       case 655: /* hint_list ::= */
+#line 1400 "sql.y"
 { yymsp[1].minor.yy748 = createHintNodeList(pCxt, NULL); }
+#line 8420 "sql.c"
         break;
       case 656: /* hint_list ::= NK_HINT */
+#line 1401 "sql.y"
 { yylhsminor.yy748 = createHintNodeList(pCxt, &yymsp[0].minor.yy0); }
+#line 8425 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 661: /* set_quantifier_opt ::= ALL */
+#line 1412 "sql.y"
 { yymsp[0].minor.yy705 = false; }
+#line 8431 "sql.c"
         break;
       case 664: /* select_item ::= NK_STAR */
+#line 1419 "sql.y"
 { yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); }
+#line 8436 "sql.c"
   yymsp[0].minor.yy600 = yylhsminor.yy600;
         break;
       case 666: /* select_item ::= common_expression column_alias */
       case 676: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==676);
+#line 1421 "sql.y"
 { yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy649); }
+#line 8443 "sql.c"
   yymsp[-1].minor.yy600 = yylhsminor.yy600;
         break;
       case 667: /* select_item ::= common_expression AS column_alias */
       case 677: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==677);
+#line 1422 "sql.y"
 { yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), &yymsp[0].minor.yy649); }
+#line 8450 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 672: /* partition_by_clause_opt ::= PARTITION BY partition_list */
       case 702: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==702);
       case 722: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==722);
+#line 1431 "sql.y"
 { yymsp[-2].minor.yy748 = yymsp[0].minor.yy748; }
+#line 8458 "sql.c"
         break;
       case 679: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA interval_sliding_duration_literal NK_RP */
+#line 1444 "sql.y"
 { yymsp[-5].minor.yy600 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 8463 "sql.c"
         break;
       case 680: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */
+#line 1445 "sql.y"
 { yymsp[-3].minor.yy600 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 8468 "sql.c"
         break;
       case 681: /* twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_RP sliding_opt fill_opt */
+#line 1447 "sql.y"
 { yymsp[-5].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 8473 "sql.c"
         break;
       case 682: /* twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_COMMA interval_sliding_duration_literal NK_RP sliding_opt fill_opt */
+#line 1451 "sql.y"
 { yymsp[-7].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); }
+#line 8478 "sql.c"
         break;
       case 683: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */
+#line 1453 "sql.y"
 { yymsp[-6].minor.yy600 = createEventWindowNode(pCxt, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); }
+#line 8483 "sql.c"
         break;
       case 684: /* twindow_clause_opt ::= COUNT_WINDOW NK_LP NK_INTEGER NK_RP */
+#line 1455 "sql.y"
 { yymsp[-3].minor.yy600 = createCountWindowNode(pCxt, &yymsp[-1].minor.yy0, &yymsp[-1].minor.yy0); }
+#line 8488 "sql.c"
         break;
       case 685: /* twindow_clause_opt ::= COUNT_WINDOW NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */
+#line 1457 "sql.y"
 { yymsp[-5].minor.yy600 = createCountWindowNode(pCxt, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0); }
+#line 8493 "sql.c"
         break;
       case 692: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */
+#line 1467 "sql.y"
 { yymsp[-3].minor.yy600 = createFillNode(pCxt, yymsp[-1].minor.yy6, NULL); }
+#line 8498 "sql.c"
         break;
       case 693: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */
+#line 1468 "sql.y"
 { yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy748)); }
+#line 8503 "sql.c"
         break;
       case 694: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */
+#line 1469 "sql.y"
 { yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy748)); }
+#line 8508 "sql.c"
         break;
       case 695: /* fill_mode ::= NONE */
+#line 1473 "sql.y"
 { yymsp[0].minor.yy6 = FILL_MODE_NONE; }
+#line 8513 "sql.c"
         break;
       case 696: /* fill_mode ::= PREV */
+#line 1474 "sql.y"
 { yymsp[0].minor.yy6 = FILL_MODE_PREV; }
+#line 8518 "sql.c"
         break;
       case 697: /* fill_mode ::= NULL */
+#line 1475 "sql.y"
 { yymsp[0].minor.yy6 = FILL_MODE_NULL; }
+#line 8523 "sql.c"
         break;
       case 698: /* fill_mode ::= NULL_F */
+#line 1476 "sql.y"
 { yymsp[0].minor.yy6 = FILL_MODE_NULL_F; }
+#line 8528 "sql.c"
         break;
       case 699: /* fill_mode ::= LINEAR */
+#line 1477 "sql.y"
 { yymsp[0].minor.yy6 = FILL_MODE_LINEAR; }
+#line 8533 "sql.c"
         break;
       case 700: /* fill_mode ::= NEXT */
+#line 1478 "sql.y"
 { yymsp[0].minor.yy6 = FILL_MODE_NEXT; }
+#line 8538 "sql.c"
         break;
       case 703: /* group_by_list ::= expr_or_subquery */
+#line 1487 "sql.y"
 { yylhsminor.yy748 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); }
+#line 8543 "sql.c"
   yymsp[0].minor.yy748 = yylhsminor.yy748;
         break;
       case 704: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */
+#line 1488 "sql.y"
 { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); }
+#line 8549 "sql.c"
   yymsp[-2].minor.yy748 = yylhsminor.yy748;
         break;
       case 708: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */
+#line 1495 "sql.y"
 { yymsp[-5].minor.yy600 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 8555 "sql.c"
         break;
       case 709: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */
+#line 1497 "sql.y"
 { yymsp[-3].minor.yy600 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); }
+#line 8560 "sql.c"
         break;
       case 712: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */
+#line 1504 "sql.y"
 {
                                                                                     yylhsminor.yy600 = addOrderByClause(pCxt, yymsp[-3].minor.yy600, yymsp[-2].minor.yy748);
                                                                                     yylhsminor.yy600 = addSlimitClause(pCxt, yylhsminor.yy600, yymsp[-1].minor.yy600);
                                                                                     yylhsminor.yy600 = addLimitClause(pCxt, yylhsminor.yy600, yymsp[0].minor.yy600);
                                                                                   }
+#line 8569 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 715: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */
+#line 1514 "sql.y"
 { yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); }
+#line 8575 "sql.c"
   yymsp[-3].minor.yy600 = yylhsminor.yy600;
         break;
       case 716: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */
+#line 1516 "sql.y"
 { yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); }
+#line 8581 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 725: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */
       case 729: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==729);
+#line 1531 "sql.y"
 { yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); }
+#line 8588 "sql.c"
         break;
       case 726: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */
       case 730: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==730);
+#line 1532 "sql.y"
 { yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); }
+#line 8594 "sql.c"
         break;
       case 731: /* subquery ::= NK_LP query_expression NK_RP */
+#line 1540 "sql.y"
 { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy600); }
+#line 8599 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 736: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */
+#line 1554 "sql.y"
 { yylhsminor.yy600 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), yymsp[-1].minor.yy1010, yymsp[0].minor.yy273); }
+#line 8605 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 737: /* ordering_specification_opt ::= */
+#line 1558 "sql.y"
 { yymsp[1].minor.yy1010 = ORDER_ASC; }
+#line 8611 "sql.c"
         break;
       case 738: /* ordering_specification_opt ::= ASC */
+#line 1559 "sql.y"
 { yymsp[0].minor.yy1010 = ORDER_ASC; }
+#line 8616 "sql.c"
         break;
       case 739: /* ordering_specification_opt ::= DESC */
+#line 1560 "sql.y"
 { yymsp[0].minor.yy1010 = ORDER_DESC; }
+#line 8621 "sql.c"
         break;
       case 740: /* null_ordering_opt ::= */
+#line 1564 "sql.y"
 { yymsp[1].minor.yy273 = NULL_ORDER_DEFAULT; }
+#line 8626 "sql.c"
         break;
       case 741: /* null_ordering_opt ::= NULLS FIRST */
+#line 1565 "sql.y"
 { yymsp[-1].minor.yy273 = NULL_ORDER_FIRST; }
+#line 8631 "sql.c"
         break;
       case 742: /* null_ordering_opt ::= NULLS LAST */
+#line 1566 "sql.y"
 { yymsp[-1].minor.yy273 = NULL_ORDER_LAST; }
+#line 8636 "sql.c"
         break;
       case 745: /* column_options ::= column_options ENCODE NK_STRING */
+#line 1574 "sql.y"
 { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_ENCODE, &yymsp[0].minor.yy0); }
+#line 8641 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 746: /* column_options ::= column_options COMPRESS NK_STRING */
+#line 1575 "sql.y"
 { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_COMPRESS, &yymsp[0].minor.yy0); }
+#line 8647 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       case 747: /* column_options ::= column_options LEVEL NK_STRING */
+#line 1576 "sql.y"
 { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_LEVEL, &yymsp[0].minor.yy0); }
+#line 8653 "sql.c"
   yymsp[-2].minor.yy600 = yylhsminor.yy600;
         break;
       default:
@@ -7680,6 +8713,7 @@ static void yy_syntax_error(
   ParseCTX_FETCH
 #define TOKEN yyminor
 /************ Begin %syntax_error code ****************************************/
+#line 29 "sql.y"
 
   if (TSDB_CODE_SUCCESS == pCxt->errCode) {
     if(TOKEN.z) {
@@ -7690,6 +8724,7 @@ static void yy_syntax_error(
   } else if (TSDB_CODE_PAR_DB_NOT_SPECIFIED == pCxt->errCode && TK_NK_FLOAT == TOKEN.type) {
     pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z);
   }
+#line 8727 "sql.c"
 /************ End %syntax_error code ******************************************/
   ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
   ParseCTX_STORE

From b56a2b5ed1ee4a6d1551d64197384df004f38484 Mon Sep 17 00:00:00 2001
From: Chris Zhai <zk662144@163.com>
Date: Fri, 31 May 2024 11:33:42 +0800
Subject: [PATCH 088/132] add test cases for TD-30189

---
 .../community/query/subquery/subqueryBugs.py  | 86 +++++++++++++++++++
 tests/parallel_test/cases.task                |  1 +
 2 files changed, 87 insertions(+)
 create mode 100644 tests/army/community/query/subquery/subqueryBugs.py

diff --git a/tests/army/community/query/subquery/subqueryBugs.py b/tests/army/community/query/subquery/subqueryBugs.py
new file mode 100644
index 0000000000..b56f4ff1b9
--- /dev/null
+++ b/tests/army/community/query/subquery/subqueryBugs.py
@@ -0,0 +1,86 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import frame.etool
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+
+
+class TDTestCase(TBase):
+    
+    def ts_30189(self):
+        tdLog.info("create database ts_30189")
+        tdSql.execute(f"create database ts_30189")
+        tdSql.execute(f"use ts_30189")
+        sqls = [
+            "CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256))",
+            "CREATE TABLE demo_201000008 USING demo (deviceid) TAGS ('201000008')",
+            "CREATE TABLE demo_K201000258 USING demo (deviceid) TAGS ('K201000258')",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-11-30 23:59:27.255', 51412.900999999998021)",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:11:28.179', 51458.900999999998021)",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:12:28.180', 51458.800999999999476)",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-31 23:59:36.108', 52855.400999999998021)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-11-30 23:59:00.365', 258839.234375000000000)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:00:00.381', 272188.843750000000000)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:01:00.600', 13.909012794494629)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-31 23:59:00.366', 1886.711303710937500)",
+        ]
+        tdSql.executes(sqls)
+        sql1 = '''
+            SELECT ts, deviceid, faev FROM (
+                (
+                    SELECT deviceid, ts, faev FROM (
+                        SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev 
+                        FROM demo 
+                        WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' 
+                        PARTITION BY deviceid 
+                    ) WHERE diff_faev < 0 
+                ) UNION ALL 
+                ( 
+                    SELECT deviceid, ts, faev FROM ( 
+                        SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev 
+                        FROM ( SELECT deviceid, _ts as ts , faev FROM demo 
+                        WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' 
+                        ORDER BY ts desc ) PARTITION BY deviceid 
+                    ) WHERE diff_faev > 0 
+                ) 
+                UNION ALL 
+                ( 
+                    SELECT deviceid, LAST(_ts) AS ts, LAST(faev) AS faev FROM demo
+                    WHERE deviceid in ('201000008') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' 
+                    PARTITION BY deviceid INTERVAL(1n) 
+                )
+            ) order by ts
+                '''
+        tdSql.query(sql1)
+        tdSql.checkRows(4)
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # TS-30189
+        self.ts_30189()
+
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 61687eeccd..12f2ee1689 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -27,6 +27,7 @@
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
 ,,n,army,python3 ./test.py -f community/query/show.py -N 3
 ,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f community/query/subquery/subqueryBugs.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3
 

From e47069b67c2fcaa1e65d751cc1f4303f17948668 Mon Sep 17 00:00:00 2001
From: Chris Zhai <zk662144@163.com>
Date: Fri, 31 May 2024 13:49:48 +0800
Subject: [PATCH 089/132] add test cases for TD-30188

---
 .../community/query/subquery/subqueryBugs.py  | 86 +++++++++++++++++++
 tests/parallel_test/cases.task                |  3 +-
 2 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 tests/army/community/query/subquery/subqueryBugs.py

diff --git a/tests/army/community/query/subquery/subqueryBugs.py b/tests/army/community/query/subquery/subqueryBugs.py
new file mode 100644
index 0000000000..b56f4ff1b9
--- /dev/null
+++ b/tests/army/community/query/subquery/subqueryBugs.py
@@ -0,0 +1,86 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import frame.etool
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+
+
+class TDTestCase(TBase):
+    
+    def ts_30189(self):
+        tdLog.info("create database ts_30189")
+        tdSql.execute(f"create database ts_30189")
+        tdSql.execute(f"use ts_30189")
+        sqls = [
+            "CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256))",
+            "CREATE TABLE demo_201000008 USING demo (deviceid) TAGS ('201000008')",
+            "CREATE TABLE demo_K201000258 USING demo (deviceid) TAGS ('K201000258')",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-11-30 23:59:27.255', 51412.900999999998021)",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:11:28.179', 51458.900999999998021)",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:12:28.180', 51458.800999999999476)",
+            "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-31 23:59:36.108', 52855.400999999998021)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-11-30 23:59:00.365', 258839.234375000000000)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:00:00.381', 272188.843750000000000)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:01:00.600', 13.909012794494629)",
+            "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-31 23:59:00.366', 1886.711303710937500)",
+        ]
+        tdSql.executes(sqls)
+        sql1 = '''
+            SELECT ts, deviceid, faev FROM (
+                (
+                    SELECT deviceid, ts, faev FROM (
+                        SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev 
+                        FROM demo 
+                        WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' 
+                        PARTITION BY deviceid 
+                    ) WHERE diff_faev < 0 
+                ) UNION ALL 
+                ( 
+                    SELECT deviceid, ts, faev FROM ( 
+                        SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev 
+                        FROM ( SELECT deviceid, _ts as ts , faev FROM demo 
+                        WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' 
+                        ORDER BY ts desc ) PARTITION BY deviceid 
+                    ) WHERE diff_faev > 0 
+                ) 
+                UNION ALL 
+                ( 
+                    SELECT deviceid, LAST(_ts) AS ts, LAST(faev) AS faev FROM demo
+                    WHERE deviceid in ('201000008') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' 
+                    PARTITION BY deviceid INTERVAL(1n) 
+                )
+            ) order by ts
+                '''
+        tdSql.query(sql1)
+        tdSql.checkRows(4)
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # TS-30189
+        self.ts_30189()
+
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 61687eeccd..8ebda24e79 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -27,7 +27,8 @@
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
 ,,n,army,python3 ./test.py -f community/query/show.py -N 3
 ,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3
-,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
+,,y,army,./pytest.sh python3 ./test.py -f community/query/subquery/subqueryBugs.py -N 3 -L 3 -D 1
+,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3
 
 #

From c253ac73a99ddbe83929bd5f59a19e525307ac21 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 06:27:54 +0000
Subject: [PATCH 090/132] support add column compress

---
 source/common/src/tcol.c               |  3 ++-
 source/libs/parser/src/parTranslater.c | 10 +++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c
index fc9dd1fb4c..ba36558587 100644
--- a/source/common/src/tcol.c
+++ b/source/common/src/tcol.c
@@ -131,6 +131,7 @@ const char* columnEncodeStr(uint8_t type) {
       encode = TSDB_COLUMN_ENCODE_DISABLED;
       break;
     default:
+      encode = TSDB_COLUMN_ENCODE_UNKNOWN;
       break;
   }
   return encode;
@@ -157,8 +158,8 @@ const char* columnCompressStr(uint16_t type) {
     case TSDB_COLVAL_COMPRESS_DISABLED:
       compress = TSDB_COLUMN_COMPRESS_DISABLED;
       break;
-
     default:
+      compress = TSDB_COLUMN_COMPRESS_UNKNOWN;
       break;
   }
   return compress;
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 938c6aae9a..a2d64d4922 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -8202,9 +8202,12 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt*
       // TAOS_FIELD        field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)};
       strcpy(field.name, pStmt->colName);
       if (pStmt->pColOptions != NULL) {
-        if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
-        if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
-        if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
+        if (!checkColumnEncodeOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->encode))
+          return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
+        if (!checkColumnCompressOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->compress))
+          return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
+        if (!checkColumnLevelOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->compressLevel))
+          return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
         int32_t code = setColCompressByOption(pStmt->dataType.type, columnEncodeVal(pStmt->pColOptions->encode),
                                               columnCompressVal(pStmt->pColOptions->compress),
                                               columnLevelVal(pStmt->pColOptions->compressLevel), false,
@@ -13049,6 +13052,7 @@ static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt,
     case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
       return buildUpdateTagValReq(pCxt, pStmt, pTableMeta, pReq);
     case TSDB_ALTER_TABLE_ADD_COLUMN:
+    case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
       return buildAddColReq(pCxt, pStmt, pTableMeta, pReq);
     case TSDB_ALTER_TABLE_DROP_COLUMN:
       return buildDropColReq(pCxt, pStmt, pTableMeta, pReq);

From 76b43dc072a455abc9772fbae256918514358dc3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 31 May 2024 14:32:55 +0800
Subject: [PATCH 091/132] fix(stream): 1. check the checkpoint-trigger rsp, 2.
 set the error code in the message body, 3. follower nodes not handle the
 checkpoint-trigger retrieve request.

---
 include/libs/stream/streammsg.h            |   1 +
 include/libs/stream/tstream.h              |   3 +-
 include/util/taoserror.h                   |   1 +
 source/dnode/vnode/src/tq/tq.c             |   9 ++
 source/dnode/vnode/src/tqCommon/tqCommon.c |  20 +++-
 source/libs/stream/inc/streamInt.h         |   1 +
 source/libs/stream/src/streamCheckpoint.c  | 124 ++++++++++++++-------
 source/libs/stream/src/streamDispatch.c    |  39 ++-----
 source/libs/stream/src/streamExec.c        |   4 -
 source/libs/stream/src/streamTask.c        |   4 +
 10 files changed, 129 insertions(+), 77 deletions(-)

diff --git a/include/libs/stream/streammsg.h b/include/libs/stream/streammsg.h
index 96701fe21d..91bfc6afc8 100644
--- a/include/libs/stream/streammsg.h
+++ b/include/libs/stream/streammsg.h
@@ -187,6 +187,7 @@ typedef struct SCheckpointTriggerRsp {
   int32_t  upstreamTaskId;
   int32_t  taskId;
   int32_t  transId;
+  int32_t  rspCode;
 } SCheckpointTriggerRsp;
 
 typedef struct {
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index d007ea29a9..0f546ee869 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -272,7 +272,6 @@ typedef struct SCheckpointInfo {
   int64_t processedVer;
   int64_t nextProcessVer;  // current offset in WAL, not serialize it
   int32_t numOfNotReady;
-
   SActiveCheckpointInfo* pActiveInfo;
   int64_t msgVer;
 } SCheckpointInfo;
@@ -678,7 +677,7 @@ bool    streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeI
 void    streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal);
 void    streamTaskInitTriggerDispatchInfo(SStreamTask* pTask);
 void    streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId);
-int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pRpcInfo);
+int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pInfo, int32_t code);
 
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
 int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue);
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 9ae75bade2..8f8434dfc1 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -910,6 +910,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_STREAM_EXEC_CANCELLED          TAOS_DEF_ERROR_CODE(0, 0x4102)
 #define TSDB_CODE_STREAM_INVALID_STATETRANS      TAOS_DEF_ERROR_CODE(0, 0x4103)
 #define TSDB_CODE_STREAM_TASK_IVLD_STATUS        TAOS_DEF_ERROR_CODE(0, 0x4104)
+#define TSDB_CODE_STREAM_NOT_LEADER              TAOS_DEF_ERROR_CODE(0, 0x4105)
 
 // TDLite
 #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS         TAOS_DEF_ERROR_CODE(0, 0x5100)
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 1e564ba467..1076f1f2c6 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1241,6 +1241,15 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
 }
 
 int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg) {
+  int32_t vgId = TD_VID(pTq->pVnode);
+
+  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont;
+  if (!vnodeIsRoleLeader(pTq->pVnode)) {
+    tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId,
+            (int32_t)pReq->downstreamTaskId);
+    return TSDB_CODE_STREAM_NOT_LEADER;
+  }
+
   return tqStreamTaskProcessRetrieveTriggerReq(pTq->pStreamMeta, pMsg);
 }
 
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 45e1a300d0..c8ca324c5e 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -877,6 +877,16 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
   tqDebug("s-task:0x%x recv retrieve checkpoint-trigger msg from downstream s-task:0x%x, checkpointId:%" PRId64,
           pReq->upstreamTaskId, (int32_t)pReq->downstreamTaskId, pReq->checkpointId);
 
+  if (pTask->status.downstreamReady != 1) {
+    tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
+            pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
+
+    streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_STREAM_TASK_IVLD_STATUS);
+    streamMetaReleaseTask(pMeta, pTask);
+
+    return TSDB_CODE_SUCCESS;
+  }
+
   SStreamTaskState* pState = streamTaskGetStatus(pTask);
   if (pState->state == TASK_STATUS__CK) {  // recv the checkpoint-source/trigger already
     int32_t transId = 0;
@@ -889,8 +899,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
       // re-send the lost checkpoint-trigger msg to downstream task
       tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
               (int32_t)pReq->downstreamTaskId, checkpointId, transId);
-      streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info);
-      return TSDB_CODE_SUCCESS;
+      streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_SUCCESS);
     } else { // not send checkpoint-trigger yet, wait
       int32_t recv = 0, total = 0;
       streamTaskGetTriggerRecvStatus(pTask, &recv, &total);
@@ -903,7 +912,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
             "sending checkpoint-source/trigger",
             pTask->id.idStr, recv, total);
       }
-      return TSDB_CODE_ACTION_IN_PROGRESS;
+      streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS);
     }
   } else {  // upstream not recv the checkpoint-source/trigger till now
     ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT);
@@ -911,8 +920,11 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
         "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
         "upstream sending checkpoint-source/trigger",
         pTask->id.idStr);
-    return TSDB_CODE_ACTION_IN_PROGRESS;
+    streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS);
   }
+
+  streamMetaReleaseTask(pMeta, pTask);
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index d6cd5b528d..9a3cbdc963 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -61,6 +61,7 @@ struct SActiveCheckpointInfo {
   bool          dispatchTrigger;
   SArray*       pDispatchTriggerList;   // SArray<STaskTriggerSendInfo>
   SArray*       pReadyMsgList;   // SArray<SStreamChkptReadyInfo*>
+  int8_t        allUpstreamTriggerRecv;
 
   int32_t       checkCounter;
   tmr_h         pCheckTmr;
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 6b02ae485f..7f41629dac 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -34,11 +34,11 @@ static int32_t streamTaskUploadCheckpoint(const char* id, const char* path);
 static int32_t deleteCheckpoint(const char* id);
 static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName);
 static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask);
-static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType);
+static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId);
 static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList);
 static void    checkpointTriggerMonitorFn(void* param, void* tmrId);
 
-static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType);
+static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId);
 
 bool streamTaskIsAllUpstreamSendTrigger(SStreamTask* pTask) {
   SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
@@ -58,7 +58,8 @@ bool streamTaskIsAllUpstreamSendTrigger(SStreamTask* pTask) {
   return allSend;
 }
 
-SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType) {
+SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId,
+    int32_t transId) {
   SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock));
   if (pChkpoint == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -75,8 +76,8 @@ SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpoint
   }
 
   pBlock->info.type = STREAM_CHECKPOINT;
-  pBlock->info.version = pTask->chkInfo.pActiveInfo->activeId;
-  pBlock->info.window.ekey = pBlock->info.window.skey = pTask->chkInfo.pActiveInfo->transId;  // NOTE: set the transId
+  pBlock->info.version = checkpointId;
+  pBlock->info.window.ekey = pBlock->info.window.skey = transId;  // NOTE: set the transId
   pBlock->info.rows = 1;
   pBlock->info.childId = pTask->info.selfChildId;
 
@@ -89,10 +90,10 @@ SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpoint
   return pChkpoint;
 }
 
-int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) {
-  SStreamDataBlock* pChkpoint = createChkptTriggerBlock(pTask, checkpointType);
+int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId) {
+  SStreamDataBlock* pCheckpoint = createChkptTriggerBlock(pTask, checkpointType, checkpointId, transId);
 
-  if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pChkpoint) < 0) {
+  if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pCheckpoint) < 0) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
@@ -116,39 +117,37 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo
 
   // 2. Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task
   // and this is the last item in the inputQ.
-  return appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER);
+  return appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pReq->checkpointId, pReq->transId);
 }
 
 int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp) {
   ASSERT(pTask->info.taskLevel != TASK_LEVEL__SOURCE);
 
-  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
-  if (pInfo->transId != pRsp->transId || pInfo->activeId != pRsp->checkpointId) {
-    // todo handle error
-    return -1;
+  if (pRsp->rspCode != TSDB_CODE_SUCCESS) {
+    stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr,
+        pRsp->upstreamTaskId, tstrerror(pRsp->rspCode));
+    return TSDB_CODE_SUCCESS;
   }
 
-  taosThreadMutexLock(&pTask->lock);
-  SStreamTaskState* pState = streamTaskGetStatus(pTask);
-  if (pState->state != TASK_STATUS__CK) {
-    // todo handle error
-    taosThreadMutexUnlock(&pTask->lock);
-    return -1;
-  }
-
-  taosThreadMutexUnlock(&pTask->lock);
-
-  appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER);
+  appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pRpcInfo) {
+int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pRpcInfo, int32_t code) {
   SCheckpointTriggerRsp* pRsp = rpcMallocCont(sizeof(SCheckpointTriggerRsp));
   pRsp->streamId = pTask->id.streamId;
   pRsp->upstreamTaskId = pTask->id.taskId;
   pRsp->taskId = dstTaskId;
-  pRsp->checkpointId = pTask->chkInfo.pActiveInfo->activeId;
-  pRsp->transId = pTask->chkInfo.pActiveInfo->transId;
+
+  if (code == TSDB_CODE_SUCCESS) {
+    pRsp->checkpointId = pTask->chkInfo.pActiveInfo->activeId;
+    pRsp->transId = pTask->chkInfo.pActiveInfo->transId;
+  } else {
+    pRsp->checkpointId = -1;
+    pRsp->transId = -1;
+  }
+
+  pRsp->rspCode = code;
 
   SRpcMsg rspMsg = {.code = 0, .pCont = pRsp, .contLen = sizeof(SCheckpointTriggerRsp), .info = *pRpcInfo};
   tmsgSendRsp(&rspMsg);
@@ -178,15 +177,64 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
   const char*  id = pTask->id.idStr;
   int32_t      code = TSDB_CODE_SUCCESS;
   int32_t      vgId = pTask->pMeta->vgId;
+  int32_t      taskLevel = pTask->info.taskLevel;
+
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+
+  taosThreadMutexLock(&pTask->lock);
+  if (pTask->chkInfo.checkpointId >= checkpointId) {
+    stError("s-task:%s vgId:%d current checkpointId:%" PRId64
+           " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard",
+           id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId);
+    taosThreadMutexUnlock(&pTask->lock);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) {
+    if (pActiveInfo->activeId != checkpointId) {
+      stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64
+              " discard",
+              id, vgId, pActiveInfo->activeId, checkpointId);
+      taosThreadMutexUnlock(&pTask->lock);
+      return TSDB_CODE_SUCCESS;
+    } else {  // checkpointId == pActiveInfo->activeId
+      if (pActiveInfo->allUpstreamTriggerRecv == 1) {
+        stDebug(
+            "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, "
+            "checkpointId:%" PRId64 " transId:%d",
+            id, vgId, checkpointId, transId);
+        taosThreadMutexUnlock(&pTask->lock);
+        return TSDB_CODE_SUCCESS;
+      }
+
+      if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
+        //  check if already recv or not, and duplicated checkpoint-trigger msg recv, discard it
+        for (int32_t i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++i) {
+          SStreamChkptReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i);
+          if (p->upStreamTaskId == pBlock->srcTaskId) {
+            ASSERT(p->checkpointId == checkpointId);
+            stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64
+                   ", prev recvTs:%" PRId64 " discard",
+                   pTask->id.idStr, p->upStreamTaskId, p->nodeId, p->checkpointId, p->recvTs);
+
+            taosThreadMutexUnlock(&pTask->lock);
+            return TSDB_CODE_SUCCESS;
+          }
+        }
+      }
+    }
+  }
+
+  taosThreadMutexUnlock(&pTask->lock);
 
   stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64
-          ", transId:%d current checkpointingId:%" PRId64,
+          ", transId:%d current active checkpointId:%" PRId64,
           id, vgId, pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, transId, checkpointId);
 
   // set task status
   if (streamTaskGetStatus(pTask)->state != TASK_STATUS__CK) {
-    pTask->chkInfo.pActiveInfo->activeId = checkpointId;
-    pTask->chkInfo.pActiveInfo->transId = transId;
+    pActiveInfo->activeId = checkpointId;
+    pActiveInfo->transId = transId;
 
     code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
     if (code != TSDB_CODE_SUCCESS) {
@@ -197,20 +245,18 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
 
     int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
     stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
-    SActiveCheckpointInfo* pActive = pTask->chkInfo.pActiveInfo;
     streamMetaAcquireOneTask(pTask);
 
-    if (pActive->pCheckTmr == NULL) {
-      pActive->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
+    if (pActiveInfo->pCheckTmr == NULL) {
+      pActiveInfo->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
     } else {
-      taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActive->pCheckTmr);
+      taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pCheckTmr);
     }
   }
 
-  // todo fix race condition: set the status and append checkpoint block
-  int32_t taskLevel = pTask->info.taskLevel;
   if (taskLevel == TASK_LEVEL__SOURCE) {
     int8_t type = pTask->outputInfo.type;
+    pActiveInfo->allUpstreamTriggerRecv = 1;
 
     if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
       stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
@@ -231,8 +277,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId);
 
     // there are still some upstream tasks not send checkpoint request, do nothing and wait for then
-    bool allSend = streamTaskIsAllUpstreamSendTrigger(pTask);
-    if (!allSend) {
+    if (pActiveInfo->allUpstreamTriggerRecv != 1) {
       streamFreeQitem((SStreamQueueItem*)pBlock);
       return code;
     }
@@ -272,7 +317,8 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) {
   if (notReady == 0) {
     stDebug("s-task:%s all downstream tasks have completed the checkpoint, start to do checkpoint for current task",
             pTask->id.idStr);
-    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT);
+    SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pInfo->activeId, pInfo->transId);
   } else {
     int32_t total = streamTaskGetNumOfDownstream(pTask);
     stDebug("s-task:%s %d/%d downstream tasks are not ready, wait", pTask->id.idStr, notReady, total);
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index a6cfbb063b..f134224196 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -559,7 +559,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
   }
 
   if (pTask->chkInfo.pActiveInfo->dispatchTrigger) {
-    stDebug("s-task:%s already send checkpoint trigger, not dispatch anymore", id);
+    stDebug("s-task:%s already send checkpoint-trigger, no longer dispatch any other data", id);
     atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
     return 0;
   }
@@ -874,29 +874,18 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
           req.upstreamNodeId);
 
   SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+
   taosThreadMutexLock(&pActiveInfo->lock);
+  taosArrayPush(pActiveInfo->pReadyMsgList, &info);
 
-  bool recved = false;
-  int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList);
-  for (int32_t i = 0; i < size; ++i) {
-    SStreamChkptReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i);
-    if (p->nodeId == req.upstreamNodeId) {
-      if (p->checkpointId == req.checkpointId) {
-        stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 ", ignore",
-               pTask->id.idStr, p->upStreamTaskId, p->nodeId, p->checkpointId);
-      } else {
-        stError("s-task:%s checkpointId:%" PRId64 " not completed, new checkpointId:%" PRId64 " recv",
-                pTask->id.idStr, p->checkpointId, checkpointId);
-        ASSERT(0); // failed to handle it
-      }
-
-      recved = true;
-      break;
-    }
-  }
-
-  if (!recved) {
-    taosArrayPush(pActiveInfo->pReadyMsgList, &info);
+  int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList);
+  int32_t total = streamTaskGetNumOfUpstream(pTask);
+  if (numOfRecv == total) {
+    stDebug("s-task:%s recv checkpoint-trigger from all upstream, continue", pTask->id.idStr);
+    pActiveInfo->allUpstreamTriggerRecv = 1;
+  } else {
+    ASSERT(numOfRecv <= total);
+    stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total);
   }
 
   taosThreadMutexUnlock(&pActiveInfo->lock);
@@ -1175,7 +1164,6 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
         // This task has received the checkpoint req from the upstream task, from which all the messages should be
         // blocked. Note that there is no race condition here.
         if (pReq->type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
-          atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
           streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId);
           stDebug("s-task:%s close inputQ for upstream:0x%x, msgId:%d", id, pReq->upstreamTaskId, pReq->msgId);
         } else if (pReq->type == STREAM_INPUT__TRANS_STATE) {
@@ -1187,11 +1175,6 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
     }
   }
 
-  // disable the data from upstream tasks
-//  if (streamTaskGetStatus(pTask)->state == TASK_STATUS__HALT) {
-//    status = TASK_INPUT_STATUS__BLOCKED;
-//  }
-
   {
     // do send response with the input status
     int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 77f266bd6d..95634b2ff3 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -597,10 +597,6 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
     // dispatch checkpoint msg to all downstream tasks
     int32_t type = pInput->type;
     if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
-      if (pTask->pMeta->vgId == 2) {
-//        taosSsleep(20);
-      }
-
       streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
       continue;
     }
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 3daadda687..01e2f89d8c 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -626,6 +626,7 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
   SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId);
   if (pInfo != NULL) {
     pInfo->dataAllowed = false;
+    int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
   }
 }
 
@@ -633,6 +634,8 @@ void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) {
   SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId);
   if (pInfo != NULL) {
     pInfo->dataAllowed = true;
+    int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
+    ASSERT(t >= 0);
   }
 }
 
@@ -1006,6 +1009,7 @@ void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) {
   pInfo->activeId = 0;  // clear the checkpoint id
   pInfo->failedId = 0;
   pInfo->transId = 0;
+  pInfo->allUpstreamTriggerRecv = 0;
   pInfo->dispatchTrigger = false;
 
   taosArrayClear(pInfo->pReadyMsgList);

From fd385d7c520ef11b5aba00ad4dd63c311ea34e2d Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 06:41:36 +0000
Subject: [PATCH 092/132] support add column compress

---
 source/libs/parser/src/parTranslater.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index a2d64d4922..1c38e1bc31 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -12898,9 +12898,9 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S
   pReq->flags = COL_SMA_ON;
   pReq->bytes = calcTypeBytes(pStmt->dataType);
   if (pStmt->pColOptions != NULL) {
-    if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
-    if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
-    if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
+    if (!checkColumnEncodeOrSetDefault(pReq->type, pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
+    if (!checkColumnCompressOrSetDefault(pReq->type, pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
+    if (!checkColumnLevelOrSetDefault(pReq->type, pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
     int8_t code = setColCompressByOption(pReq->type, columnEncodeVal(pStmt->pColOptions->encode),
                                          columnCompressVal(pStmt->pColOptions->compress),
                                          columnLevelVal(pStmt->pColOptions->compressLevel), true, &pReq->compress);

From b8e63fa5f4707848748a1b23f02718edc7e9e103 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 06:56:33 +0000
Subject: [PATCH 093/132] support add column compress

---
 tests/script/tsim/compress/compress_col.sim | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/script/tsim/compress/compress_col.sim b/tests/script/tsim/compress/compress_col.sim
index cc5b46fa06..ddf65b7f40 100644
--- a/tests/script/tsim/compress/compress_col.sim
+++ b/tests/script/tsim/compress/compress_col.sim
@@ -21,4 +21,15 @@ sql create table txx2 (ts timestamp compress 'disabled' level 'h', f int compres
 sql create table txx3 (ts timestamp compress 'disabled' level 'h', f int compress 'xz')
 sql create table txx4 (ts timestamp compress 'disabled' level 'h', f int compress 'zstd')
 
+
+sql alter table txx4 add column tt int compress 'xz' level 'h'
+sql alter table txx4 drop column tt 
+
+sql create stable sp(ts timestamp, c int) tags(t int)
+sql alter table sp add column c1 int compress 'zstd' 
+sql alter table sp drop column c1 
+
+sql alter stable sp add column c1 int compress 'zstd' 
+sql alter stable sp drop column c1 
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From a77c6940da711b11eb85d3186b9e48357156c5b0 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 31 May 2024 15:22:38 +0800
Subject: [PATCH 094/132] when dst table has composite primary key, need to
 delete result

---
 source/libs/executor/src/streamtimewindowoperator.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 2da9ed0353..46844a2470 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -1427,7 +1427,9 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);
   }
 
-  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);
+  if (IS_FINAL_INTERVAL_OP(pOperator) && !pInfo->destHasPrimaryKey) {
+    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);
+  }
   if (IS_FINAL_INTERVAL_OP(pOperator)) {
     closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,
                               pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator);
@@ -2845,7 +2847,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);
   closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
   copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);
-  removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);
+  if (!pInfo->destHasPrimaryKey) {
+    removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);
+  }
   if (pInfo->isHistoryOp) {
     getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);
   }
@@ -4131,7 +4135,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
     pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);
   }
   pOperator->status = OP_RES_TO_RETURN;
-  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);
+  if (!pInfo->destHasPrimaryKey) {
+    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);
+  }
   closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL,
                             pInfo->pUpdatedMap, pInfo->pDelWins, pOperator);
   if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) {

From e7c8952202afe8054048d1fb2bf9223ded5054be Mon Sep 17 00:00:00 2001
From: liuyao <54liuyao@163.com>
Date: Fri, 31 May 2024 16:05:32 +0800
Subject: [PATCH 095/132] Update 14-stream.md

---
 docs/zh/12-taos-sql/14-stream.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index 5691716ce8..c94df0a7ba 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -190,7 +190,7 @@ T3 时刻，最新事件到达，T 向后推移超过了第二个窗口关闭的
 
 TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项指定：
 
-1. 重新计算，即 IGNORE EXPIRED 0：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果
+1. 增量计算，即 IGNORE EXPIRED 0。
 
 2. 直接丢弃，即 IGNORE EXPIRED 1：默认配置，忽略过期数据
 

From 9fb329f481c96e8ec419c477788238917aa9f9a7 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 08:08:54 +0000
Subject: [PATCH 096/132] add error

---
 source/libs/transport/src/transCli.c | 33 ++++++++++++++++------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index d0fb5cbfdb..9a29444d2d 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -200,7 +200,7 @@ static FORCE_INLINE void cliMayUpdateFqdnCache(SHashObj* cache, char* dst);
 // process data read from server, add decompress etc later
 static void cliHandleResp(SCliConn* conn);
 // handle except about conn
-static void cliHandleExcept(SCliConn* conn);
+static void cliHandleExcept(SCliConn* conn, int32_t code);
 static void cliReleaseUnfinishedMsg(SCliConn* conn);
 static void cliHandleFastFail(SCliConn* pConn, int status);
 
@@ -571,8 +571,11 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
   if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn);
   transUnrefCliHandle(pConn);
 }
-void cliHandleExcept(SCliConn* conn) {
+void cliHandleExcept(SCliConn* conn, int32_t code) {
   tTrace("%s conn %p except ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn));
+  if (code != TSDB_CODE_RPC_FQDN_ERROR) {
+    code = -1;
+  }
   cliHandleExceptImpl(conn, -1);
 }
 
@@ -866,7 +869,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
     while (transReadComplete(pBuf)) {
       tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn);
       if (pBuf->invalid) {
-        cliHandleExcept(conn);
+        cliHandleExcept(conn, -1);
         break;
       } else {
         cliHandleResp(conn);
@@ -886,7 +889,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
     tDebug("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
            T_REF_VAL_GET(conn));
     conn->broken = true;
-    cliHandleExcept(conn);
+    cliHandleExcept(conn, -1);
   }
 }
 
@@ -1037,7 +1040,7 @@ static void cliSendCb(uv_write_t* req, int status) {
   } else {
     if (!uv_is_closing((uv_handle_t*)&pConn->stream)) {
       tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status));
-      cliHandleExcept(pConn);
+      cliHandleExcept(pConn, -1);
     }
     return;
   }
@@ -1113,7 +1116,7 @@ void cliSend(SCliConn* pConn) {
 
   if (transQueueEmpty(&pConn->cliMsgs)) {
     tError("%s conn %p not msg to send", pTransInst->label, pConn);
-    cliHandleExcept(pConn);
+    cliHandleExcept(pConn, -1);
     return;
   }
 
@@ -1186,7 +1189,7 @@ void cliSend(SCliConn* pConn) {
   if (status != 0) {
     tGError("%s conn %p failed to send msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType),
             uv_err_name(status));
-    cliHandleExcept(pConn);
+    cliHandleExcept(pConn, -1);
   }
   return;
 _RETURN:
@@ -1242,7 +1245,8 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) {
       taosArrayPush(pThrd->timerList, &conn->timer);
       conn->timer = NULL;
 
-      cliHandleFastFail(conn, -1);
+      cliHandleFastFail(conn, terrno);
+      terrno = 0;
       return;
     }
     struct sockaddr_in addr;
@@ -1304,7 +1308,7 @@ static void cliSendBatchCb(uv_write_t* req, int status) {
     tDebug("%s conn %p failed to send batch msg, batch size:%d, msgLen:%d, reason:%s", CONN_GET_INST_LABEL(conn), conn,
            p->wLen, p->batchSize, uv_err_name(status));
 
-    if (!uv_is_closing((uv_handle_t*)&conn->stream)) cliHandleExcept(conn);
+    if (!uv_is_closing((uv_handle_t*)&conn->stream)) cliHandleExcept(conn, -1);
 
     cliHandleBatchReq(nxtBatch, thrd);
   } else {
@@ -1362,7 +1366,7 @@ static void cliHandleFastFail(SCliConn* pConn, int status) {
     cliDestroyBatch(pConn->pBatch);
     pConn->pBatch = NULL;
   }
-  cliHandleExcept(pConn);
+  cliHandleExcept(pConn, status);
 }
 
 void cliConnCb(uv_connect_t* req, int status) {
@@ -1653,7 +1657,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
       taosArrayPush(pThrd->timerList, &conn->timer);
       conn->timer = NULL;
 
-      cliHandleExcept(conn);
+      cliHandleExcept(conn, terrno);
+      terrno = 0;
       return;
     }
 
@@ -1667,20 +1672,20 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
     if (fd == -1) {
       tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn,
               tstrerror(TAOS_SYSTEM_ERROR(errno)));
-      cliHandleExcept(conn);
+      cliHandleExcept(conn, -1);
       errno = 0;
       return;
     }
     int ret = uv_tcp_open((uv_tcp_t*)conn->stream, fd);
     if (ret != 0) {
       tGError("%s conn %p failed to set stream, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret));
-      cliHandleExcept(conn);
+      cliHandleExcept(conn, -1);
       return;
     }
     ret = transSetConnOption((uv_tcp_t*)conn->stream, tsKeepAliveIdle);
     if (ret != 0) {
       tGError("%s conn %p failed to set socket opt, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret));
-      cliHandleExcept(conn);
+      cliHandleExcept(conn, -1);
       return;
     }
 

From 426527023fd56b126a7d2765c81d50b407211971 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Fri, 31 May 2024 15:56:17 +0800
Subject: [PATCH 097/132] TD-30354: update test case

---
 tests/system-test/2-query/avg.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py
index dd8b38f9ea..34e4a19978 100644
--- a/tests/system-test/2-query/avg.py
+++ b/tests/system-test/2-query/avg.py
@@ -436,13 +436,13 @@ class TDTestCase:
         tdSql.execute(f"use db")
         tdSql.execute(f"create table db.st(ts timestamp, ibv bigint, ubv bigint unsigned) tags(area int)")
         # insert t1 data
-        tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,9223372036854775801,18446744073709551611)")
-        tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,8223372036854775801,17446744073709551611)")
-        tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,7223372036854775801,16446744073709551611)")
+        tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 1s,9223372036854775801,18446744073709551611)")
+        tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 2s,8223372036854775801,17446744073709551611)")
+        tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 3s,7223372036854775801,16446744073709551611)")
         # insert t2 data
-        tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,9223372036854775801,18446744073709551611)")
-        tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,8223372036854775801,17446744073709551611)")
-        tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,7223372036854775801,16446744073709551611)")
+        tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 1s,9223372036854775801,18446744073709551611)")
+        tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 2s,8223372036854775801,17446744073709551611)")
+        tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 3s,7223372036854775801,16446744073709551611)")
         
         # check single table answer
         tdSql.query(f"select avg(ibv), avg(ubv) from db.t1")

From 923eaaa569a4729efa0ac0de99ca2f0f3e89fde3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 31 May 2024 16:12:44 +0800
Subject: [PATCH 098/132] fix(test): update test cases.

---
 tests/system-test/0-others/information_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py
index 66a69c4f9a..e8ed3db1b9 100644
--- a/tests/system-test/0-others/information_schema.py
+++ b/tests/system-test/0-others/information_schema.py
@@ -222,7 +222,7 @@ class TDTestCase:
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
         tdLog.info(len(tdSql.queryResult))
-        tdSql.checkEqual(True, len(tdSql.queryResult) in range(259, 260))
+        tdSql.checkEqual(True, len(tdSql.queryResult) in range(260, 261))
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
         tdSql.checkEqual(54, len(tdSql.queryResult))

From 2d9f81d34c4e689f32b0831c9c4be08c77be61c0 Mon Sep 17 00:00:00 2001
From: Chris Zhai <zk662144@163.com>
Date: Fri, 31 May 2024 16:49:03 +0800
Subject: [PATCH 099/132] add verification of query date

---
 tests/army/community/query/subquery/subqueryBugs.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/army/community/query/subquery/subqueryBugs.py b/tests/army/community/query/subquery/subqueryBugs.py
index b56f4ff1b9..8442474d40 100644
--- a/tests/army/community/query/subquery/subqueryBugs.py
+++ b/tests/army/community/query/subquery/subqueryBugs.py
@@ -70,6 +70,14 @@ class TDTestCase(TBase):
         tdSql.query(sql1)
         tdSql.checkRows(4)
 
+        row1 = ['2023-11-30 23:59:27.255', "201000008", 51412.900999999998021]
+        row2 = ['2023-12-04 23:11:28.179', "201000008", 51458.900999999998021]
+        row3 = ['2023-12-04 23:12:28.180', "201000008", 51458.800999999999476]
+        row4 = ['2023-12-31 23:59:36.108', "201000008", 52855.400999999998021]
+
+        rows = [row1, row2, row3, row4]
+        tdSql.checkDataMem(sql1, rows)
+
     # run
     def run(self):
         tdLog.debug(f"start to excute {__file__}")

From e0167511c1a2c10f9c06c1c6a7ac4d17ed441adb Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 31 May 2024 17:18:25 +0800
Subject: [PATCH 100/132] use tDeleteSVCreateTbBatchReq free mem

---
 source/client/src/clientRawBlockWrite.c | 17 ++---------------
 source/common/src/tmsg.c                |  7 +++++--
 source/dnode/vnode/src/tq/tqRead.c      |  9 +--------
 source/dnode/vnode/src/vnd/vnodeSvr.c   |  7 +------
 4 files changed, 9 insertions(+), 31 deletions(-)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 5e6d693562..d8bd56985f 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -440,14 +440,7 @@ static cJSON* processCreateTable(SMqMetaRsp* metaRsp) {
 
 _exit:
   uDebug("create table return, sql json:%s", cJSON_PrintUnformatted(pJson));
-  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
-    pCreateReq = req.pReqs + iReq;
-    taosMemoryFreeClear(pCreateReq->comment);
-    taosMemoryFreeClear(pCreateReq->sql);
-    if (pCreateReq->type == TSDB_CHILD_TABLE) {
-      taosArrayDestroy(pCreateReq->ctb.tagName);
-    }
-  }
+  tDeleteSVCreateTbBatchReq(&req);
   tDecoderClear(&decoder);
   return pJson;
 }
@@ -1110,13 +1103,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
 
 end:
   uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code));
-  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
-    pCreateReq = req.pReqs + iReq;
-    taosMemoryFreeClear(pCreateReq->comment);
-    if (pCreateReq->type == TSDB_CHILD_TABLE) {
-      taosArrayDestroy(pCreateReq->ctb.tagName);
-    }
-  }
+  tDeleteSVCreateTbBatchReq(&req);
 
   taosHashCleanup(pVgroupHashmap);
   destroyRequest(pRequest);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index c17c522d4f..6bb88b7824 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -8369,11 +8369,14 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) {
   return 0;
 }
 
-void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq* pReq) {
+void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq *pReq) {
   for (int32_t iReq = 0; iReq < pReq->nReqs; iReq++) {
-    SVCreateTbReq* pCreateReq = pReq->pReqs + iReq;
+    SVCreateTbReq *pCreateReq = pReq->pReqs + iReq;
+    taosMemoryFreeClear(pCreateReq->sql);
+    taosMemoryFreeClear(pCreateReq->comment);
     if (pCreateReq->type == TSDB_CHILD_TABLE) {
       taosArrayDestroy(pCreateReq->ctb.tagName);
+      pCreateReq->ctb.tagName = NULL;
     }
   }
 }
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 9cbe48a0b0..40b817accd 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -99,14 +99,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
       taosArrayDestroy(reqNew.pArray);
     }
 
-    for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
-      pCreateReq = req.pReqs + iReq;
-      taosMemoryFreeClear(pCreateReq->comment);
-      taosMemoryFreeClear(pCreateReq->sql);
-      if (pCreateReq->type == TSDB_CHILD_TABLE) {
-        taosArrayDestroy(pCreateReq->ctb.tagName);
-      }
-    }
+    tDeleteSVCreateTbBatchReq(&req);
   } else if (msgType == TDMT_VND_ALTER_TABLE) {
     SVAlterTbReq req = {0};
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index cb71495e9f..3a3109862c 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -1158,12 +1158,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
   }
 
 _exit:
-  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
-    pCreateReq = req.pReqs + iReq;
-    taosMemoryFree(pCreateReq->sql);
-    taosMemoryFree(pCreateReq->comment);
-    taosArrayDestroy(pCreateReq->ctb.tagName);
-  }
+  tDeleteSVCreateTbBatchReq(&req);
   taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp);
   taosArrayDestroy(tbUids);
   tDecoderClear(&decoder);

From 5cbd733cab1911ec3a336bdf59d58dbcd855dfe5 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 31 May 2024 18:41:20 +0800
Subject: [PATCH 101/132] fix(util): idle 500ms after kill rsync

---
 source/common/src/rsync.c | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
index 1b03b142e0..2ed21616dc 100644
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@@ -114,20 +114,24 @@ static int32_t execCommand(char* command){
 void stopRsync() {
   int32_t code =
 #ifdef WINDOWS
-  system("taskkill /f /im rsync.exe");
+      system("taskkill /f /im rsync.exe");
 #else
-  system("pkill rsync");
+      system("pkill rsync");
 #endif
-  if(code != 0){
-    uError("[rsync] stop rsync server failed,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
-    return;
+
+  if (code != 0) {
+    uError("[rsync] stop rsync server failed," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
+  } else {
+    uDebug("[rsync] stop rsync server successful");
   }
-  uDebug("[rsync] stop rsync server successful");
+
+  taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
 }
 
 void startRsync() {
-  if(taosMulMkDir(tsCheckpointBackupDir) != 0){
-    uError("[rsync] build checkpoint backup dir failed, dir:%s,"ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA);
+  if (taosMulMkDir(tsCheckpointBackupDir) != 0) {
+    uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir,
+           ERRNO_ERR_DATA);
     return;
   }
 
@@ -137,7 +141,7 @@ void startRsync() {
   snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir);
 
   int32_t code = generateConfigFile(confDir);
-  if(code != 0){
+  if (code != 0) {
     return;
   }
 
@@ -145,12 +149,12 @@ void startRsync() {
   snprintf(cmd, PATH_MAX, "rsync --daemon --port=%d --config=%s", tsRsyncPort, confDir);
   // start rsync service to backup checkpoint
   code = system(cmd);
-  if(code != 0){
-    uError("[rsync] start server failed, code:%d,"ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
-    return;
+  if (code != 0) {
+    uError("[rsync] start server failed, code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+  } else {
+    uDebug("[rsync] start server successful");
   }
 
-  uDebug("[rsync] start server successful");
 }
 
 int32_t uploadByRsync(const char* id, const char* path) {

From 9f0d6eddaeade9cbc9ea2b1d11f57fe514999d53 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 09:34:41 +0000
Subject: [PATCH 102/132] add error

---
 source/libs/parser/src/parAstCreater.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 86ce93d920..13c74ca881 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -1907,8 +1907,12 @@ SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRea
   COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName);
   pStmt->dataType = dataType;
 
+  // if pOptions is NULL, it means that the column has no options
   if (pOptions != NULL) {
-    pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION;
+    SColumnOptions* pOption = (SColumnOptions*)pOptions;
+    if (strlen(pOption->compress) != 0 || strlen(pOption->compressLevel) || strlen(pOption->encode) != 0) {
+      pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION;
+    }
   }
   pStmt->pColOptions = (SColumnOptions*)pOptions;
   return createAlterTableStmtFinalize(pRealTable, pStmt);

From 69e402d547998953aef833f7d86d14b01f83719e Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 10:36:30 +0000
Subject: [PATCH 103/132] add error

---
 source/libs/parser/src/parAstCreater.c | 18 ++++++++++++++----
 source/libs/parser/src/parTranslater.c |  6 ++++--
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 13c74ca881..00ce03cf6a 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -1901,20 +1901,30 @@ SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRea
   if (!checkColumnName(pCxt, pColName)) {
     return NULL;
   }
+
   SAlterTableStmt* pStmt = (SAlterTableStmt*)nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   pStmt->alterType = alterType;
   COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName);
   pStmt->dataType = dataType;
+  pStmt->pColOptions = (SColumnOptions*)pOptions;
 
-  // if pOptions is NULL, it means that the column has no options
   if (pOptions != NULL) {
     SColumnOptions* pOption = (SColumnOptions*)pOptions;
-    if (strlen(pOption->compress) != 0 || strlen(pOption->compressLevel) || strlen(pOption->encode) != 0) {
-      pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION;
+    if (pOption->bPrimaryKey == false || pOption->commentNull == true) {
+      if (strlen(pOption->compress) != 0 || strlen(pOption->compressLevel) || strlen(pOption->encode) != 0) {
+        pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION;
+      } else {
+        pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
+                                                "not support alter column with option except compress");
+        return NULL;
+      }
+    } else {
+      pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
+                                              "not support alter column with option except compress");
+      return NULL;
     }
   }
-  pStmt->pColOptions = (SColumnOptions*)pOptions;
   return createAlterTableStmtFinalize(pRealTable, pStmt);
 }
 
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1c38e1bc31..b6673d6219 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -12899,8 +12899,10 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S
   pReq->bytes = calcTypeBytes(pStmt->dataType);
   if (pStmt->pColOptions != NULL) {
     if (!checkColumnEncodeOrSetDefault(pReq->type, pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
-    if (!checkColumnCompressOrSetDefault(pReq->type, pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
-    if (!checkColumnLevelOrSetDefault(pReq->type, pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
+    if (!checkColumnCompressOrSetDefault(pReq->type, pStmt->pColOptions->compress))
+      return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR;
+    if (!checkColumnLevelOrSetDefault(pReq->type, pStmt->pColOptions->compressLevel))
+      return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR;
     int8_t code = setColCompressByOption(pReq->type, columnEncodeVal(pStmt->pColOptions->encode),
                                          columnCompressVal(pStmt->pColOptions->compress),
                                          columnLevelVal(pStmt->pColOptions->compressLevel), true, &pReq->compress);

From ac2d9ab276461bccd5c63f9ac445de10ac9389e7 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Fri, 31 May 2024 11:38:55 +0000
Subject: [PATCH 104/132] add error

---
 source/libs/parser/src/parAstCreater.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 00ce03cf6a..0e31672b24 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -1911,13 +1911,13 @@ SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRea
 
   if (pOptions != NULL) {
     SColumnOptions* pOption = (SColumnOptions*)pOptions;
-    if (pOption->bPrimaryKey == false || pOption->commentNull == true) {
+    if (pOption->bPrimaryKey == false && pOption->commentNull == true) {
       if (strlen(pOption->compress) != 0 || strlen(pOption->compressLevel) || strlen(pOption->encode) != 0) {
         pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION;
       } else {
-        pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
-                                                "not support alter column with option except compress");
-        return NULL;
+        // pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
+        //                                         "not support alter column with option except compress");
+        // return NULL;
       }
     } else {
       pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,

From ad075ef1dec28e22a640687b46c610f0c113c851 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Sat, 1 Jun 2024 16:19:17 +0800
Subject: [PATCH 105/132] fix:[TD-30323]tmq close error if consumer is cleared

---
 source/dnode/mnode/impl/src/mndConsumer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 9278fbce11..72a8c32056 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -662,7 +662,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
   SMqConsumerObj *pConsumerNew = NULL;
   STrans         *pTrans = NULL;
 
-  if(mndAcquireConsumer(pMnode, subscribe->consumerId) == NULL && taosArrayGetSize(subscribe->topicNames) == 0){
+  if(mndAcquireConsumer(pMnode, subscribe.consumerId) == NULL && taosArrayGetSize(subscribe.topicNames) == 0){
     goto _over;
   }
 

From 2a8270f9c835e55969f9f241e2c9bc4fa33eec48 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 1 Jun 2024 18:26:45 +0800
Subject: [PATCH 106/132] fix(stream): track the checkpoint-ready msg on the
 upstream tasks.

---
 include/libs/stream/tstream.h              |  2 +-
 source/dnode/vnode/src/tqCommon/tqCommon.c |  2 +-
 source/libs/stream/inc/streamInt.h         | 20 ++++--
 source/libs/stream/src/streamCheckpoint.c  | 81 +++++++++++++++++-----
 source/libs/stream/src/streamDispatch.c    | 57 ++++++++-------
 source/libs/stream/src/streamTask.c        |  5 +-
 6 files changed, 115 insertions(+), 52 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 0f546ee869..99ca2104ff 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -753,7 +753,7 @@ tmr_h   streamTimerGetInstance();
 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp);
-int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
+int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNodeId, int32_t downstreamTaskId);
 int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
 int32_t streamAlignTransferState(SStreamTask* pTask);
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index c8ca324c5e..b779cbe932 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -492,7 +492,7 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
   tqDebug("vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", vgId,
           pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
 
-  streamProcessCheckpointReadyMsg(pTask);
+  streamProcessCheckpointReadyMsg(pTask, req.downstreamTaskId, req.downstreamNodeId);
   streamMetaReleaseTask(pMeta, pTask);
 
   {  // send checkpoint ready rsp
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 9a3cbdc963..c943f663e6 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -60,9 +60,9 @@ struct SActiveCheckpointInfo {
   int64_t       failedId;
   bool          dispatchTrigger;
   SArray*       pDispatchTriggerList;   // SArray<STaskTriggerSendInfo>
-  SArray*       pReadyMsgList;   // SArray<SStreamChkptReadyInfo*>
+  SArray*       pReadyMsgList;   // SArray<STaskCheckpointReadyInfo*>
   int8_t        allUpstreamTriggerRecv;
-
+  SArray*       pCheckpointReadyRecvList;   // SArray<STaskCheckpointReadyRecvInfo>
   int32_t       checkCounter;
   tmr_h         pCheckTmr;
 };
@@ -97,14 +97,14 @@ struct STokenBucket {
 };
 
 typedef struct {
-  int32_t upStreamTaskId;
+  int32_t upstreamTaskId;
   SEpSet  upstreamNodeEpset;
   int32_t nodeId;
   SRpcMsg msg;
   int64_t recvTs;
   int32_t transId;
   int64_t checkpointId;
-} SStreamChkptReadyInfo;
+} STaskCheckpointReadyInfo;
 
 typedef struct {
   int64_t sendTs;
@@ -114,6 +114,15 @@ typedef struct {
   int32_t taskId;
 } STaskTriggerSendInfo;
 
+typedef struct {
+  int64_t streamId;
+  int64_t recvTs;
+  int32_t downstreamNodeId;
+  int32_t downstreamTaskId;
+  int64_t checkpointId;
+  int32_t transId;
+} STaskCheckpointReadyRecvInfo;
+
 struct SStreamQueue {
   STaosQueue* pQueue;
   STaosQall*  qall;
@@ -203,6 +212,9 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path);
 int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
 int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask);
 
+int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, SStreamTask* pTask, int32_t upstreamNodeId,
+                                int32_t upstreamTaskId, int32_t childId, SEpSet* pEpset, int64_t checkpointId);
+
 typedef int32_t (*__stream_async_exec_fn_t)(void* param);
 
 int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code);
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 7f41629dac..6b6b740f01 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -182,7 +182,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
   SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
 
   taosThreadMutexLock(&pTask->lock);
-  if (pTask->chkInfo.checkpointId >= checkpointId) {
+  if (pTask->chkInfo.checkpointId > checkpointId) {
     stError("s-task:%s vgId:%d current checkpointId:%" PRId64
            " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard",
            id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId);
@@ -190,6 +190,26 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     return TSDB_CODE_SUCCESS;
   }
 
+  if (pTask->chkInfo.checkpointId == checkpointId) {
+    {  // send checkpoint-ready msg to upstream
+      SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId);
+
+      STaskCheckpointReadyInfo info = {0};
+      initCheckpointReadyInfo(&info, pTask, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId);
+
+      tmsgSendReq(&info.upstreamNodeEpset, &info.msg);
+    }
+
+    stWarn(
+        "s-task:%s vgId:%d recv already finished checkpoint msg, send checkpoint-ready to upstream:0x%x to resume the "
+        "interrupted checkpoint",
+        id, vgId, pBlock->srcTaskId);
+
+    streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId);
+    taosThreadMutexUnlock(&pTask->lock);
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) {
     if (pActiveInfo->activeId != checkpointId) {
       stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64
@@ -210,12 +230,12 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
         //  check if already recv or not, and duplicated checkpoint-trigger msg recv, discard it
         for (int32_t i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++i) {
-          SStreamChkptReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i);
-          if (p->upStreamTaskId == pBlock->srcTaskId) {
+          STaskCheckpointReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i);
+          if (p->upstreamTaskId == pBlock->srcTaskId) {
             ASSERT(p->checkpointId == checkpointId);
             stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64
                    ", prev recvTs:%" PRId64 " discard",
-                   pTask->id.idStr, p->upStreamTaskId, p->nodeId, p->checkpointId, p->recvTs);
+                   pTask->id.idStr, p->upstreamTaskId, p->nodeId, p->checkpointId, p->recvTs);
 
             taosThreadMutexUnlock(&pTask->lock);
             return TSDB_CODE_SUCCESS;
@@ -262,8 +282,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
       continueDispatchCheckpointTriggerBlock(pBlock, pTask);
     } else {  // only one task exists, no need to dispatch downstream info
-      atomic_add_fetch_32(&pTask->chkInfo.numOfNotReady, 1);
-      streamProcessCheckpointReadyMsg(pTask);
+      streamProcessCheckpointReadyMsg(pTask, 0, 0);
       streamFreeQitem((SStreamQueueItem*)pBlock);
     }
   } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@@ -307,23 +326,47 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
  * All down stream tasks have successfully completed the check point task.
  * Current stream task is allowed to start to do checkpoint things in ASYNC model.
  */
-int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) {
+int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNodeId, int32_t downstreamTaskId) {
   ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG);
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+
+  const char* id = pTask->id.idStr;
+  bool        received = false;
+  int32_t     total = streamTaskGetNumOfDownstream(pTask);
+
+  taosThreadMutexLock(&pInfo->lock);
 
   // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task
-  int32_t notReady = atomic_sub_fetch_32(&pTask->chkInfo.numOfNotReady, 1);
-  ASSERT(notReady >= 0);
-
-  if (notReady == 0) {
-    stDebug("s-task:%s all downstream tasks have completed the checkpoint, start to do checkpoint for current task",
-            pTask->id.idStr);
-    SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
-    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pInfo->activeId, pInfo->transId);
-  } else {
-    int32_t total = streamTaskGetNumOfDownstream(pTask);
-    stDebug("s-task:%s %d/%d downstream tasks are not ready, wait", pTask->id.idStr, notReady, total);
+  int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
+  for (int32_t i = 0; i < size; ++i) {
+    STaskCheckpointReadyRecvInfo* p = taosArrayGet(pInfo->pCheckpointReadyRecvList, i);
+    if (p->downstreamTaskId == downstreamTaskId) {
+      received = true;
+      break;
+    }
   }
 
+  if (received) {
+    stDebug("s-task:%s already recv checkpoint-ready msg from downstream:0x%x, %d/%d downstream not ready", id,
+            downstreamTaskId, (int32_t)(total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList)), total);
+  } else {
+    STaskCheckpointReadyRecvInfo info = {.recvTs = taosGetTimestampMs(),
+                                         .downstreamTaskId = downstreamTaskId,
+                                         .checkpointId = pInfo->activeId,
+                                         .transId = pInfo->transId,
+                                         .streamId = pTask->id.streamId,
+                                         .downstreamNodeId = downstreamNodeId};
+    taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
+  }
+
+  int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
+  if (notReady == 0) {
+    stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task",
+            id);
+    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pInfo->activeId, pInfo->transId);
+  }
+
+  taosThreadMutexUnlock(&pInfo->lock);
   return 0;
 }
 
@@ -685,7 +728,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
 
     bool recved = false;
     for(int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) {
-      SStreamChkptReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j);
+      STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j);
       if (pInfo->nodeId == pReady->nodeId) {
         recved = true;
         break;
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index f134224196..aa0d7c3120 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -633,11 +633,11 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
   ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
 
   for (int32_t i = 0; i < num; ++i) {
-    SStreamChkptReadyInfo* pInfo = taosArrayGet(pList, i);
+    STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
     tmsgSendReq(&pInfo->upstreamNodeEpset, &pInfo->msg);
 
     stDebug("s-task:%s level:%d checkpoint ready msg sent to upstream:0x%x", pTask->id.idStr, pTask->info.taskLevel,
-            pInfo->upStreamTaskId);
+            pInfo->upstreamTaskId);
   }
 
   taosArrayClear(pList);
@@ -657,7 +657,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
   ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
 
   if (taosArrayGetSize(pList) == 1) {
-    SStreamChkptReadyInfo* pInfo = taosArrayGet(pList, 0);
+    STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, 0);
     tmsgSendRsp(&pInfo->msg);
 
     taosArrayClear(pList);
@@ -785,7 +785,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
 }
 
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask) {
-  SStreamChkptReadyInfo info = {
+  STaskCheckpointReadyInfo info = {
       .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId};
 
   streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS);
@@ -797,7 +797,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
   if (size > 0) {
     ASSERT(size == 1);
 
-    SStreamChkptReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0);
+    STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0);
     if (pReady->transId == pReq->transId) {
       stWarn("s-task:%s repeatly recv checkpoint source msg from mnode, checkpointId:%" PRId64 ", ignore",
              pTask->id.idStr, pReq->checkpointId);
@@ -816,24 +816,20 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) {
+int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, SStreamTask* pTask, int32_t upstreamNodeId,
+                                  int32_t upstreamTaskId, int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
   int32_t code = 0;
   int32_t tlen = 0;
   void*   buf = NULL;
-  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
 
   SStreamCheckpointReadyMsg req = {0};
   req.downstreamNodeId = pTask->pMeta->vgId;
   req.downstreamTaskId = pTask->id.taskId;
   req.streamId = pTask->id.streamId;
   req.checkpointId = checkpointId;
-  req.childId = pInfo->childId;
-  req.upstreamNodeId = pInfo->nodeId;
-  req.upstreamTaskId = pInfo->taskId;
+  req.childId = childId;
+  req.upstreamNodeId = upstreamNodeId;
+  req.upstreamTaskId = upstreamTaskId;
 
   tEncodeSize(tEncodeStreamCheckpointReadyMsg, &req, tlen, code);
   if (code < 0) {
@@ -858,20 +854,29 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
 
   ASSERT(req.upstreamTaskId != 0);
 
-  SStreamChkptReadyInfo info = {
-      .upStreamTaskId = pInfo->taskId,
-      .upstreamNodeEpset = pInfo->epSet,
-      .nodeId = req.upstreamNodeId,
-      .recvTs = taosGetTimestampMs(),
-      .checkpointId = req.checkpointId,
-  };
+  pReadyInfo->upstreamTaskId = upstreamTaskId;
+  pReadyInfo->upstreamNodeEpset = *pEpset;
+  pReadyInfo->nodeId = req.upstreamNodeId;
+  pReadyInfo->recvTs = taosGetTimestampMs();
+  pReadyInfo->checkpointId = req.checkpointId;
 
-  initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead));
+  initRpcMsg(&pReadyInfo->msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead));
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) {
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
+
+  STaskCheckpointReadyInfo info = {0};
+  initCheckpointReadyInfo(&info, pTask, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId);
 
   stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64
-          ":0x%x (vgId:%d) idx:%d, vgId:%d",
-          pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index,
-          req.upstreamNodeId);
+          "-0x%x (vgId:%d) idx:%d",
+          pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index);
 
   SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
 
@@ -899,7 +904,7 @@ void streamClearChkptReadyMsg(SStreamTask* pTask) {
   }
 
   for (int i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); i++) {
-    SStreamChkptReadyInfo* pInfo = taosArrayGet(pActiveInfo->pReadyMsgList, i);
+    STaskCheckpointReadyInfo* pInfo = taosArrayGet(pActiveInfo->pReadyMsgList, i);
     rpcFreeCont(pInfo->msg.pCont);
   }
 
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 01e2f89d8c..8084a978ef 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -984,7 +984,8 @@ SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo() {
   taosThreadMutexInit(&pInfo->lock, NULL);
 
   pInfo->pDispatchTriggerList = taosArrayInit(4, sizeof(STaskTriggerSendInfo));
-  pInfo->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo));
+  pInfo->pReadyMsgList = taosArrayInit(4, sizeof(STaskCheckpointReadyInfo));
+  pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskCheckpointReadyRecvInfo));
   return pInfo;
 }
 
@@ -996,6 +997,7 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
   taosThreadMutexDestroy(&pInfo->lock);
   pInfo->pDispatchTriggerList = taosArrayDestroy(pInfo->pDispatchTriggerList);
   pInfo->pReadyMsgList = taosArrayDestroy(pInfo->pReadyMsgList);
+  pInfo->pCheckpointReadyRecvList = taosArrayDestroy(pInfo->pCheckpointReadyRecvList);
 
   if (pInfo->pCheckTmr != NULL) {
     taosTmrStop(pInfo->pCheckTmr);
@@ -1014,4 +1016,5 @@ void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) {
 
   taosArrayClear(pInfo->pReadyMsgList);
   taosArrayClear(pInfo->pDispatchTriggerList);
+  taosArrayClear(pInfo->pCheckpointReadyRecvList);
 }
\ No newline at end of file

From 31317c489508aae81268fdb0e5a95e3c4eb227f8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 1 Jun 2024 23:20:43 +0800
Subject: [PATCH 107/132] fix(stream): set the checkpoint ready info for only
 one task in stream.

---
 source/libs/stream/inc/streamInt.h        |  9 +++++----
 source/libs/stream/src/streamCheckpoint.c | 21 ++++++++++++++-------
 source/libs/stream/src/streamTask.c       |  2 +-
 3 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index c943f663e6..c4c3298ea7 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -62,7 +62,7 @@ struct SActiveCheckpointInfo {
   SArray*       pDispatchTriggerList;   // SArray<STaskTriggerSendInfo>
   SArray*       pReadyMsgList;   // SArray<STaskCheckpointReadyInfo*>
   int8_t        allUpstreamTriggerRecv;
-  SArray*       pCheckpointReadyRecvList;   // SArray<STaskCheckpointReadyRecvInfo>
+  SArray*       pCheckpointReadyRecvList;   // SArray<STaskDownstreamReadyInfo>
   int32_t       checkCounter;
   tmr_h         pCheckTmr;
 };
@@ -100,10 +100,11 @@ typedef struct {
   int32_t upstreamTaskId;
   SEpSet  upstreamNodeEpset;
   int32_t nodeId;
-  SRpcMsg msg;
-  int64_t recvTs;
   int32_t transId;
+  SRpcMsg msg;
   int64_t checkpointId;
+  int64_t recvTs;
+  int32_t sendToUpstream;
 } STaskCheckpointReadyInfo;
 
 typedef struct {
@@ -121,7 +122,7 @@ typedef struct {
   int32_t downstreamTaskId;
   int64_t checkpointId;
   int32_t transId;
-} STaskCheckpointReadyRecvInfo;
+} STaskDownstreamReadyInfo;
 
 struct SStreamQueue {
   STaosQueue* pQueue;
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 6b6b740f01..f2868fea96 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -334,12 +334,19 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNo
   bool        received = false;
   int32_t     total = streamTaskGetNumOfDownstream(pTask);
 
+  // only one task in this stream
+  if (total == 0 && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pInfo->activeId, pInfo->transId);
+    taosThreadMutexUnlock(&pInfo->lock);
+    return 0;
+  }
+
   taosThreadMutexLock(&pInfo->lock);
 
   // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task
   int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
   for (int32_t i = 0; i < size; ++i) {
-    STaskCheckpointReadyRecvInfo* p = taosArrayGet(pInfo->pCheckpointReadyRecvList, i);
+    STaskDownstreamReadyInfo* p = taosArrayGet(pInfo->pCheckpointReadyRecvList, i);
     if (p->downstreamTaskId == downstreamTaskId) {
       received = true;
       break;
@@ -350,12 +357,12 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNo
     stDebug("s-task:%s already recv checkpoint-ready msg from downstream:0x%x, %d/%d downstream not ready", id,
             downstreamTaskId, (int32_t)(total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList)), total);
   } else {
-    STaskCheckpointReadyRecvInfo info = {.recvTs = taosGetTimestampMs(),
-                                         .downstreamTaskId = downstreamTaskId,
-                                         .checkpointId = pInfo->activeId,
-                                         .transId = pInfo->transId,
-                                         .streamId = pTask->id.streamId,
-                                         .downstreamNodeId = downstreamNodeId};
+    STaskDownstreamReadyInfo info = {.recvTs = taosGetTimestampMs(),
+                                     .downstreamTaskId = downstreamTaskId,
+                                     .checkpointId = pInfo->activeId,
+                                     .transId = pInfo->transId,
+                                     .streamId = pTask->id.streamId,
+                                     .downstreamNodeId = downstreamNodeId};
     taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
   }
 
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 8084a978ef..f6524a69ab 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -985,7 +985,7 @@ SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo() {
 
   pInfo->pDispatchTriggerList = taosArrayInit(4, sizeof(STaskTriggerSendInfo));
   pInfo->pReadyMsgList = taosArrayInit(4, sizeof(STaskCheckpointReadyInfo));
-  pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskCheckpointReadyRecvInfo));
+  pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo));
   return pInfo;
 }
 

From a096e2c071b9defe1e39e273f1e6b9203208a37b Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Sun, 2 Jun 2024 21:28:53 +0800
Subject: [PATCH 108/132] fix:memory leak

---
 source/dnode/mnode/impl/src/mndConsumer.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 72a8c32056..3eef2afcc1 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -662,8 +662,12 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
   SMqConsumerObj *pConsumerNew = NULL;
   STrans         *pTrans = NULL;
 
-  if(mndAcquireConsumer(pMnode, subscribe.consumerId) == NULL && taosArrayGetSize(subscribe.topicNames) == 0){
-    goto _over;
+  if(taosArrayGetSize(subscribe.topicNames) == 0){
+    SMqConsumerObj *pConsumerTmp = mndAcquireConsumer(pMnode, subscribe.consumerId);
+    if(pConsumerTmp == NULL){
+      goto _over;
+    }
+    mndReleaseConsumer(pMnode, pConsumerTmp);
   }
 
   code = checkAndSortTopic(pMnode, subscribe.topicNames);

From d85d3aa70e66e997074cd9b3b34255d52b77c4a4 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Sun, 2 Jun 2024 22:06:52 +0800
Subject: [PATCH 109/132] fix:[TD-30306]error in converity scan

---
 source/client/src/clientTmq.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 0bc4214c09..7bc6a2f2e6 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -858,7 +858,10 @@ void tmqSendHbReq(void* param, void* tmrId) {
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
   int64_t transporterId = 0;
-  asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
+  int32_t code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
+  if (code != 0) {
+    tscError("tmqSendHbReq asyncSendMsgToServer failed");
+  }
 
 OVER:
   tDestroySMqHbReq(&req);
@@ -1220,12 +1223,15 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
   if (sendInfo == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(buf);
     goto FAIL;
   }
 
   SMqSubscribeCbParam param = {.rspErr = 0};
   if (tsem_init(&param.rspSem, 0, 0) != 0) {
     code = TSDB_CODE_TSC_INTERNAL_ERROR;
+    taosMemoryFree(buf);
+    taosMemoryFree(sendInfo);
     goto FAIL;
   }
 
@@ -1245,10 +1251,6 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
     goto FAIL;
   }
 
-  // avoid double free if msg is sent
-  buf = NULL;
-  sendInfo = NULL;
-
   tsem_wait(&param.rspSem);
   tsem_destroy(&param.rspSem);
 
@@ -1284,8 +1286,6 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
 
 FAIL:
   taosArrayDestroyP(req.topicNames, taosMemoryFree);
-  taosMemoryFree(buf);
-  taosMemoryFree(sendInfo);
 
   return code;
 }
@@ -1745,6 +1745,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
   pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
   if (pParam == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFreeClear(msg);
     goto FAIL;
   }
 
@@ -1756,6 +1757,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
   sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
   if (sendInfo == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFreeClear(msg);
     goto FAIL;
   }
 
@@ -1782,7 +1784,6 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
 
   return 0;
 FAIL:
-  taosMemoryFreeClear(msg);
   return tmqPollCb(pParam, NULL, code);
 }
 
@@ -2560,6 +2561,7 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
   if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) {
     tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen);
     code = TSDB_CODE_INVALID_PARA;
+    taosMemoryFree(pReq);
     goto FAIL;
   }
 
@@ -2567,6 +2569,7 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
   if (pParam == NULL) {
     tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId);
     code = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pReq);
     goto FAIL;
   }
 
@@ -2577,6 +2580,7 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
   SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
   if (sendInfo == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pReq);
     goto FAIL;
   }
 
@@ -2598,8 +2602,6 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
   }
 
 FAIL:
-  taosMemoryFreeClear(pParam);
-  taosMemoryFreeClear(pReq);
   askEpCb(pParam, NULL, code);
 }
 
@@ -2706,7 +2708,6 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
 
   taosMemoryFree(pMsg->pData);
   taosMemoryFree(pMsg->pEpSet);
-  taosMemoryFree(pParam);
   return 0;
 }
 
@@ -3066,6 +3067,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
       sendInfo->requestId = req.reqId;
       sendInfo->requestObjRefId = 0;
       sendInfo->param = pParam;
+      sendInfo->paramFreeFp = taosMemoryFree;
       sendInfo->fp = tmqGetWalInfoCb;
       sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO;
 
@@ -3077,8 +3079,6 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
               tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
       code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo);
       if (code != 0) {
-        taosMemoryFree(pParam);
-        taosMemoryFree(msg);
         goto end;
       }
     }

From f13dc1858ab7cac08983546ac01a686fb40a0113 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 3 Jun 2024 09:54:28 +0800
Subject: [PATCH 110/132] fix(stream): disable stream checkpoint when related
 fill-history task exists.

---
 source/dnode/mnode/impl/src/mndStream.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index b88c4a4665..db4b345536 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1108,6 +1108,15 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) {
       ready = false;
       break;
     }
+
+    if (pEntry->hTaskId != 0) {
+      mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s related fill-history task:0x%" PRIx64
+             " exists, checkpoint not issued",
+             pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status),
+             pEntry->hTaskId);
+      ready = false;
+      break;
+    }
   }
 
   taosThreadMutexUnlock(&execInfo.lock);

From cf9cd211f5daa4bbce9ee8dc167a613927c1d397 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 3 Jun 2024 13:36:16 +0800
Subject: [PATCH 111/132] fix: possible coredump

---
 source/common/src/tdataformat.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index 8d3db72fb5..fb97c011fa 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -383,7 +383,9 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
           if (IS_VAR_DATA_TYPE(schema->columns[i].type)) {
             payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
             payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData);
-            memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
+            if (colValArray[colValIndex].value.nData > 0) {
+              memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
+            }
             payloadSize += colValArray[colValIndex].value.nData;
           } else {
             payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);

From 6fcd0711447a5c8873544ba997065efdffe17f56 Mon Sep 17 00:00:00 2001
From: wangjiaming0909 <604227650@qq.com>
Date: Fri, 31 May 2024 10:08:15 +0000
Subject: [PATCH 112/132] fix exchange operator memory access problem

---
 include/os/osAtomic.h                       | 6 ++++++
 source/libs/executor/src/exchangeoperator.c | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h
index 48b7b8c56f..97bb6f53ba 100644
--- a/include/os/osAtomic.h
+++ b/include/os/osAtomic.h
@@ -114,6 +114,12 @@ int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val);
 void   *atomic_fetch_xor_ptr(void *ptr, void *val);
 
+#ifdef _MSC_VER
+#define tmemory_barrier(order) MemoryBarrier()
+#else
+#define tmemory_barrier(order) __sync_synchronize()
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index 06dd43e170..631a92f1be 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -93,6 +93,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
         goto _error;
       }
 
+      tmemory_barrier();
       SRetrieveTableRsp*     pRsp = pDataInfo->pRsp;
       SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pDataInfo->index);
 
@@ -428,6 +429,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
     }
   }
 
+  tmemory_barrier();
   pSourceDataInfo->status = EX_SOURCE_DATA_READY;
   code = tsem_post(&pExchangeInfo->ready);
   if (code != TSDB_CODE_SUCCESS) {

From ca7e1344e192887568b1ddd503e8ed47fb19d816 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Mon, 3 Jun 2024 15:20:05 +0800
Subject: [PATCH 113/132] adj stream count state buff

---
 source/libs/stream/src/streamSessionState.c | 52 ++++++++++++++-------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index 687b4bcf12..4c61e6da1d 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -734,6 +734,21 @@ _end:
   return code;
 }
 
+int32_t getCountWinStateFromDisc(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
+  SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext_rocksdb(pState, pKey);
+  int32_t          code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen);
+  streamStateFreeCur(pCur);
+  if (code == TSDB_CODE_SUCCESS) {
+    return code;
+  } else {
+    pCur = streamStateSessionSeekKeyPrev_rocksdb(pState, pKey);
+  }
+
+  code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen);
+  streamStateFreeCur(pCur);
+  return code;
+}
+
 int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) {
   SSessionKey* pWinKey = pKey;
   const TSKEY gap = 0;
@@ -755,14 +770,13 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
   if (size == 0) {
     void* pFileStore = getStateFileStore(pFileState);
     void* pRockVal = NULL;
-    SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId);
-    code = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &pRockVal, pVLen);
-    streamStateFreeCur(pCur);
+    code = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen);
     if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
-      qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code);
+      qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey,
+             pWinKey->win.ekey, code);
       if (code == TSDB_CODE_SUCCESS) {
         int32_t     valSize = *pVLen;
-        COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)) );
+        COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)));
         if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) {
           (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen);
           goto _end;
@@ -798,20 +812,24 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
   }
 
   if (index == -1) {
-    if (!isDeteled(pFileState, endTs)) {
-      void*   p = NULL;
-      void*   pFileStore = getStateFileStore(pFileState);
-      SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId);
-      int32_t code_file = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &p, pVLen);
+    if (!isDeteled(pFileState, endTs) && isFlushedState(pFileState, endTs, 0)) {
+      SSessionKey tmpKey = *pWinKey;
+      void*       pRockVal = NULL;
+      void*       pFileStore = getStateFileStore(pFileState);
+      int32_t     code_file = getCountWinStateFromDisc(pFileStore, &tmpKey, &pRockVal, pVLen);
       if (code_file == TSDB_CODE_SUCCESS) {
-        (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen);
-        code = code_file;
-        qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file);
-        streamStateFreeCur(pCur);
-        goto _end;
+        SRowBuffPos* pFirstPos = taosArrayGetP(pWinStates, 0);
+        SSessionKey* pFirstWinKey = (SSessionKey*)pFirstPos->pKey;
+        if (tmpKey.win.ekey < pFirstWinKey->win.skey) {
+          *pWinKey = tmpKey;
+          (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen);
+          code = code_file;
+          qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey,
+                 pWinKey->win.ekey, code_file);
+          goto _end;
+        }
       }
-      taosMemoryFree(p);
-      streamStateFreeCur(pCur);
+      taosMemoryFree(pRockVal);
     }
   }
 

From 1ad25599cac8a16d42a60e69356cab39ead893f1 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Mon, 3 Jun 2024 16:41:38 +0800
Subject: [PATCH 114/132] init delete mark

---
 .../libs/executor/src/streamcountwindowoperator.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c
index 050e67e15d..857d048457 100644
--- a/source/libs/executor/src/streamcountwindowoperator.c
+++ b/source/libs/executor/src/streamcountwindowoperator.c
@@ -677,6 +677,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
     goto _error;
   }
 
+  pInfo->twAggSup = (STimeWindowAggSupp){
+      .waterMark = pCountNode->window.watermark,
+      .calTrigger = pCountNode->window.triggerType,
+      .maxTs = INT64_MIN,
+      .minTs = INT64_MAX,
+      .deleteMark = getDeleteMark(&pCountNode->window, 0),
+  };
+
   pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId;
   code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0,
                                 pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle,
@@ -687,13 +695,6 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->streamAggSup.windowCount = pCountNode->windowCount;
   pInfo->streamAggSup.windowSliding = pCountNode->windowSliding;
 
-  pInfo->twAggSup = (STimeWindowAggSupp){
-      .waterMark = pCountNode->window.watermark,
-      .calTrigger = pCountNode->window.triggerType,
-      .maxTs = INT64_MIN,
-      .minTs = INT64_MAX,
-  };
-
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
 
   pInfo->binfo.pRes = pResBlock;

From 3b3ed1c30c067ac5e647a13e55a4debc0c00b529 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 3 Jun 2024 16:45:37 +0800
Subject: [PATCH 115/132] fix(stream): add monitor for checkpoint-ready msg
 recv status.

---
 include/libs/stream/tstream.h              |   4 +-
 source/dnode/vnode/src/tq/tq.c             |   9 +
 source/dnode/vnode/src/tqCommon/tqCommon.c |  40 +++-
 source/libs/stream/inc/streamInt.h         |  17 +-
 source/libs/stream/src/streamCheckpoint.c  |  88 ++++++---
 source/libs/stream/src/streamDispatch.c    | 203 +++++++++++++++------
 source/libs/stream/src/streamMeta.c        |   5 +-
 source/libs/stream/src/streamTask.c        |   7 +-
 8 files changed, 273 insertions(+), 100 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 99ca2104ff..26c6631ee4 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -271,7 +271,6 @@ typedef struct SCheckpointInfo {
   int64_t checkpointTime;  // latest checkpoint time
   int64_t processedVer;
   int64_t nextProcessVer;  // current offset in WAL, not serialize it
-  int32_t numOfNotReady;
   SActiveCheckpointInfo* pActiveInfo;
   int64_t msgVer;
 } SCheckpointInfo;
@@ -753,7 +752,8 @@ tmr_h   streamTimerGetInstance();
 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp);
-int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNodeId, int32_t downstreamTaskId);
+int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, int32_t downstreamTaskId);
+int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId);
 int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
 int32_t streamAlignTransferState(SStreamTask* pTask);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 1076f1f2c6..712cfbaa55 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1229,6 +1229,15 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 
 // downstream task has complete the stream task checkpoint procedure, let's start the handle the rsp by execute task
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg) {
+  int32_t vgId = TD_VID(pTq->pVnode);
+
+  SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont;
+  if (!vnodeIsRoleLeader(pTq->pVnode)) {
+    tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId,
+            (int32_t)pReq->downstreamTaskId);
+    return TSDB_CODE_STREAM_NOT_LEADER;
+  }
+
   return tqStreamTaskProcessCheckpointReadyMsg(pTq->pStreamMeta, pMsg);
 }
 
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index b779cbe932..c55745e5c5 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -19,6 +19,13 @@
 
 typedef struct SMStreamCheckpointReadyRspMsg {
   SMsgHead head;
+  int64_t streamId;
+  int32_t upstreamTaskId;
+  int32_t upstreamNodeId;
+  int32_t downstreamTaskId;
+  int32_t downstreamNodeId;
+  int64_t checkpointId;
+  int32_t transId;
 } SMStreamCheckpointReadyRspMsg;
 
 static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
@@ -486,21 +493,27 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId);
   if (pTask == NULL) {
     tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId);
-    return code;
+    return TSDB_CODE_STREAM_TASK_NOT_EXIST;
   }
 
-  tqDebug("vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", vgId,
+  tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId,
           pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
 
-  streamProcessCheckpointReadyMsg(pTask, req.downstreamTaskId, req.downstreamNodeId);
+  streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId);
   streamMetaReleaseTask(pMeta, pTask);
 
   {  // send checkpoint ready rsp
-    SRpcMsg rsp = {.code = 0, .info = pMsg->info, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)};
-    rsp.pCont = rpcMallocCont(rsp.contLen);
-    SMsgHead* pHead = rsp.pCont;
-    pHead->vgId = htonl(req.downstreamNodeId);
+    SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg));
 
+    pReadyRsp->upstreamTaskId = req.upstreamTaskId;
+    pReadyRsp->upstreamNodeId = req.upstreamNodeId;
+    pReadyRsp->downstreamTaskId = req.downstreamTaskId;
+    pReadyRsp->downstreamNodeId = req.downstreamNodeId;
+    pReadyRsp->checkpointId = req.checkpointId;
+    pReadyRsp->streamId = req.streamId;
+    pReadyRsp->head.vgId = htonl(req.downstreamNodeId);
+
+    SRpcMsg rsp = {.code = 0, .info = pMsg->info, .pCont = pReadyRsp, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)};
     tmsgSendRsp(&rsp);
 
     pMsg->info.handle = NULL;  // disable auto rsp
@@ -1066,5 +1079,16 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d
 int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
 
 int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
-  return doProcessDummyRspMsg(pMeta, pMsg);
+  SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;
+
+  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId);
+  if (pTask == NULL) {
+    tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
+            pRsp->downstreamNodeId, pRsp->downstreamTaskId);
+    return TSDB_CODE_STREAM_TASK_NOT_EXIST;
+  }
+
+  streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId);
+  streamMetaReleaseTask(pMeta, pTask);
+  return TSDB_CODE_SUCCESS;
 }
\ No newline at end of file
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index c4c3298ea7..154f623b9d 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -64,7 +64,9 @@ struct SActiveCheckpointInfo {
   int8_t        allUpstreamTriggerRecv;
   SArray*       pCheckpointReadyRecvList;   // SArray<STaskDownstreamReadyInfo>
   int32_t       checkCounter;
-  tmr_h         pCheckTmr;
+  tmr_h         pChkptTriggerTmr;
+  int32_t       sendReadyCheckCounter;
+  tmr_h         pSendReadyMsgTmr;
 };
 
 typedef struct {
@@ -99,12 +101,13 @@ struct STokenBucket {
 typedef struct {
   int32_t upstreamTaskId;
   SEpSet  upstreamNodeEpset;
-  int32_t nodeId;
+  int32_t upstreamNodeId;
   int32_t transId;
-  SRpcMsg msg;
+  int32_t childId;
+  SRpcMsg msg;                 // for mnode checkpoint-source rsp
   int64_t checkpointId;
   int64_t recvTs;
-  int32_t sendToUpstream;
+  int32_t sendCompleted;
 } STaskCheckpointReadyInfo;
 
 typedef struct {
@@ -213,8 +216,10 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path);
 int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
 int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask);
 
-int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, SStreamTask* pTask, int32_t upstreamNodeId,
-                                int32_t upstreamTaskId, int32_t childId, SEpSet* pEpset, int64_t checkpointId);
+int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
+                                int32_t childId, SEpSet* pEpset, int64_t checkpointId);
+int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId,
+                               int64_t checkpointId, SRpcMsg* pMsg);
 
 typedef int32_t (*__stream_async_exec_fn_t)(void* param);
 
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index f2868fea96..b0c4884c73 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -111,7 +111,6 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo
 
   pTask->chkInfo.pActiveInfo->transId = pReq->transId;
   pTask->chkInfo.pActiveInfo->activeId = pReq->checkpointId;
-  pTask->chkInfo.numOfNotReady = streamTaskGetNumOfDownstream(pTask);
   pTask->chkInfo.startTs = taosGetTimestampMs();
   pTask->execInfo.checkpoint += 1;
 
@@ -192,12 +191,11 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
 
   if (pTask->chkInfo.checkpointId == checkpointId) {
     {  // send checkpoint-ready msg to upstream
+      SRpcMsg msg ={0};
+
       SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId);
-
-      STaskCheckpointReadyInfo info = {0};
-      initCheckpointReadyInfo(&info, pTask, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId);
-
-      tmsgSendReq(&info.upstreamNodeEpset, &info.msg);
+      initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg);
+      tmsgSendReq(&pInfo->epSet, &msg);
     }
 
     stWarn(
@@ -235,7 +233,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
             ASSERT(p->checkpointId == checkpointId);
             stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64
                    ", prev recvTs:%" PRId64 " discard",
-                   pTask->id.idStr, p->upstreamTaskId, p->nodeId, p->checkpointId, p->recvTs);
+                   pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs);
 
             taosThreadMutexUnlock(&pTask->lock);
             return TSDB_CODE_SUCCESS;
@@ -267,10 +265,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
     stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
     streamMetaAcquireOneTask(pTask);
 
-    if (pActiveInfo->pCheckTmr == NULL) {
-      pActiveInfo->pCheckTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
+    if (pActiveInfo->pChkptTriggerTmr == NULL) {
+      pActiveInfo->pChkptTriggerTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
     } else {
-      taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pCheckTmr);
+      taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr);
     }
   }
 
@@ -282,7 +280,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
       continueDispatchCheckpointTriggerBlock(pBlock, pTask);
     } else {  // only one task exists, no need to dispatch downstream info
-      streamProcessCheckpointReadyMsg(pTask, 0, 0);
+      appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId);
       streamFreeQitem((SStreamQueueItem*)pBlock);
     }
   } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@@ -308,11 +306,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
       streamTaskBuildCheckpoint(pTask);
     } else {  // source & agg tasks need to forward the checkpoint msg downwards
       stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num);
-
-      // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task
-      // can start local checkpoint procedure
-      pTask->chkInfo.numOfNotReady = streamTaskGetNumOfDownstream(pTask);
-
       // Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task
       // already. And then, dispatch check point msg to all downstream tasks
       code = continueDispatchCheckpointTriggerBlock(pBlock, pTask);
@@ -326,19 +319,31 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
  * All down stream tasks have successfully completed the check point task.
  * Current stream task is allowed to start to do checkpoint things in ASYNC model.
  */
-int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNodeId, int32_t downstreamTaskId) {
+int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId,
+                                        int32_t downstreamTaskId) {
   ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG);
   SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
 
   const char* id = pTask->id.idStr;
   bool        received = false;
   int32_t     total = streamTaskGetNumOfDownstream(pTask);
+  ASSERT(total > 0);
 
-  // only one task in this stream
-  if (total == 0 && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pInfo->activeId, pInfo->transId);
-    taosThreadMutexUnlock(&pInfo->lock);
-    return 0;
+  // 1. not in checkpoint status now
+  SStreamTaskState* pStat = streamTaskGetStatus(pTask);
+  if (pStat->state != TASK_STATUS__CK) {
+    return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
+  }
+
+  // 2. expired checkpoint-ready msg
+  if (pTask->chkInfo.checkpointId > checkpointId) {
+    // discard it directly
+    return -1;
+  }
+
+  // invalid checkpoint-ready msg
+  if (pInfo->activeId != checkpointId) {
+    return -1;
   }
 
   taosThreadMutexLock(&pInfo->lock);
@@ -354,7 +359,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNo
   }
 
   if (received) {
-    stDebug("s-task:%s already recv checkpoint-ready msg from downstream:0x%x, %d/%d downstream not ready", id,
+    stDebug("s-task:%s already recv checkpoint-ready msg from downstream:0x%x, ignore. %d/%d downstream not ready", id,
             downstreamTaskId, (int32_t)(total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList)), total);
   } else {
     STaskDownstreamReadyInfo info = {.recvTs = taosGetTimestampMs(),
@@ -377,9 +382,38 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int32_t downstreamNo
   return 0;
 }
 
+int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId) {
+  SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
+  int64_t now = taosGetTimestampMs();
+  int32_t numOfConfirmed = 0;
+
+  taosThreadMutexLock(&pInfo->lock);
+  for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) {
+    STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i);
+    if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) {
+      pReadyInfo->sendCompleted = 1;
+      stDebug("s-task:%s send checkpoint-ready msg to upstream:0x%x confirmed, checkpointId:%" PRId64 " ts:%" PRId64,
+              pTask->id.idStr, upstreamTaskId, checkpointId, now);
+      break;
+    }
+  }
+
+  for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) {
+    STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i);
+    if (pReadyInfo->sendCompleted == 1) {
+      numOfConfirmed += 1;
+    }
+  }
+
+  stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr,
+          numOfConfirmed, checkpointId);
+
+  taosThreadMutexUnlock(&pInfo->lock);
+  return TSDB_CODE_SUCCESS;
+}
+
 void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
   pTask->chkInfo.startTs = 0;  // clear the recorded start time
-  pTask->chkInfo.numOfNotReady = 0;
 
   streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo);
   streamTaskOpenAllUpstreamInput(pTask);  // open inputQ for all upstream tasks
@@ -703,7 +737,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
   }
 
   if (++pActiveInfo->checkCounter < 100) {
-    taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pCheckTmr);
+    taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr);
     return;
   }
 
@@ -736,7 +770,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
     bool recved = false;
     for(int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) {
       STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j);
-      if (pInfo->nodeId == pReady->nodeId) {
+      if (pInfo->nodeId == pReady->upstreamNodeId) {
         recved = true;
         break;
       }
@@ -756,7 +790,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
   // check every 100ms
   if (size > 0) {
     stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id);
-    taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pCheckTmr);
+    taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr);
   } else {
     int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
     stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref);
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index aa0d7c3120..42a4e4e8fb 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -623,28 +623,163 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
-// this function is usually invoked by sink/agg task
-int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
-  SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList;
+int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId,
+                               int64_t checkpointId, SRpcMsg* pMsg) {
+  int32_t code = 0;
+  int32_t tlen = 0;
+  void*   buf = NULL;
 
-  taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock);
+  SStreamCheckpointReadyMsg req = {0};
+  req.downstreamNodeId = pTask->pMeta->vgId;
+  req.downstreamTaskId = pTask->id.taskId;
+  req.streamId = pTask->id.streamId;
+  req.checkpointId = checkpointId;
+  req.childId = childId;
+  req.upstreamNodeId = upstreamNodeId;
+  req.upstreamTaskId = upstreamTaskId;
+
+  tEncodeSize(tEncodeStreamCheckpointReadyMsg, &req, tlen, code);
+  if (code < 0) {
+    return -1;
+  }
+
+  buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
+  if (buf == NULL) {
+    return -1;
+  }
+
+  ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId);
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+
+  SEncoder encoder;
+  tEncoderInit(&encoder, abuf, tlen);
+  if ((code = tEncodeStreamCheckpointReadyMsg(&encoder, &req)) < 0) {
+    rpcFreeCont(buf);
+    return code;
+  }
+  tEncoderClear(&encoder);
+
+  initRpcMsg(pMsg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead));
+  return TSDB_CODE_SUCCESS;
+}
+
+static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
+  SStreamTask* pTask = param;
+  int32_t      vgId = pTask->pMeta->vgId;
+  const char*  id = pTask->id.idStr;
+
+  // check the status every 100ms
+  if (streamTaskShouldStop(pTask)) {
+    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref);
+    streamMetaReleaseTask(pTask->pMeta, pTask);
+    return;
+  }
+
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+  if (++pActiveInfo->sendReadyCheckCounter < 100) {
+    taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr);
+    return;
+  }
+
+  pActiveInfo->sendReadyCheckCounter = 0;
+  stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id);
+
+  taosThreadMutexLock(&pActiveInfo->lock);
+
+  SArray* pList = pActiveInfo->pReadyMsgList;
+  SArray* pNotRspList = taosArrayInit(4, sizeof(int32_t));
 
   int32_t num = taosArrayGetSize(pList);
   ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
 
   for (int32_t i = 0; i < num; ++i) {
     STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
-    tmsgSendReq(&pInfo->upstreamNodeEpset, &pInfo->msg);
+    if (pInfo->sendCompleted == 1) {
+      continue;
+    }
 
-    stDebug("s-task:%s level:%d checkpoint ready msg sent to upstream:0x%x", pTask->id.idStr, pTask->info.taskLevel,
+    taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
+    stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId,
+            pTask->info.taskLevel, pInfo->upstreamTaskId);
+  }
+
+  int32_t checkpointId = pActiveInfo->activeId;
+
+  int32_t notRsp = taosArrayGetSize(pNotRspList);
+  if (notRsp > 0) { // send checkpoint-ready msg again
+    for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) {
+      int32_t taskId = *(int32_t*)taosArrayGet(pNotRspList, i);
+
+      for (int32_t j = 0; j < num; ++j) {
+        STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pList, j);
+        if (taskId == pReadyInfo->upstreamTaskId) { // send msg again
+
+          SRpcMsg msg = {0};
+          initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId,
+                                 checkpointId, &msg);
+          tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg);
+          stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel,
+                  pReadyInfo->upstreamTaskId);
+        }
+      }
+    }
+
+    taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr);
+    taosThreadMutexUnlock(&pActiveInfo->lock);
+  } else {
+    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+    stDebug(
+        "s-task:%s vgId:%d recv of checkpoint-ready msg confirmed by all upstream task(s), quit from timer and clear "
+        "checkpoint-ready msg, ref:%d",
+        id, vgId, ref);
+
+    streamClearChkptReadyMsg(pTask);
+    taosThreadMutexUnlock(&pActiveInfo->lock);
+    streamMetaReleaseTask(pTask->pMeta, pTask);
+  }
+
+  taosArrayDestroy(pNotRspList);
+}
+
+// this function is usually invoked by sink/agg task
+int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
+  SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
+
+  const char* id = pTask->id.idStr;
+  SArray*     pList = pActiveInfo->pReadyMsgList;
+
+  taosThreadMutexLock(&pActiveInfo->lock);
+
+  int32_t num = taosArrayGetSize(pList);
+  ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
+
+  for (int32_t i = 0; i < num; ++i) {
+    STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
+
+    SRpcMsg msg = {0};
+    initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, &msg);
+    tmsgSendReq(&pInfo->upstreamNodeEpset, &msg);
+
+    stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel,
             pInfo->upstreamTaskId);
   }
 
-  taosArrayClear(pList);
+  taosThreadMutexUnlock(&pActiveInfo->lock);
+  stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num);
 
-  taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
-  stDebug("s-task:%s level:%d checkpoint ready msg sent to all %d upstreams", pTask->id.idStr, pTask->info.taskLevel,
-          num);
+  // start to check if checkpoint ready msg has successfully received by upstream tasks.
+  pActiveInfo->pSendReadyMsgTmr = NULL;
+
+  int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
+  stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
+  streamMetaAcquireOneTask(pTask);
+
+  if (pActiveInfo->pSendReadyMsgTmr == NULL) {
+    pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer);
+  } else {
+    taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr);
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -816,51 +951,17 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, SStreamTask* pTask, int32_t upstreamNodeId,
-                                  int32_t upstreamTaskId, int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
-  int32_t code = 0;
-  int32_t tlen = 0;
-  void*   buf = NULL;
-
-  SStreamCheckpointReadyMsg req = {0};
-  req.downstreamNodeId = pTask->pMeta->vgId;
-  req.downstreamTaskId = pTask->id.taskId;
-  req.streamId = pTask->id.streamId;
-  req.checkpointId = checkpointId;
-  req.childId = childId;
-  req.upstreamNodeId = upstreamNodeId;
-  req.upstreamTaskId = upstreamTaskId;
-
-  tEncodeSize(tEncodeStreamCheckpointReadyMsg, &req, tlen, code);
-  if (code < 0) {
-    return -1;
-  }
-
-  buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
-  if (buf == NULL) {
-    return -1;
-  }
-
-  ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId);
-  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-
-  SEncoder encoder;
-  tEncoderInit(&encoder, abuf, tlen);
-  if ((code = tEncodeStreamCheckpointReadyMsg(&encoder, &req)) < 0) {
-    rpcFreeCont(buf);
-    return code;
-  }
-  tEncoderClear(&encoder);
-
-  ASSERT(req.upstreamTaskId != 0);
+int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
+                                int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
+  ASSERT(upstreamTaskId != 0);
 
   pReadyInfo->upstreamTaskId = upstreamTaskId;
   pReadyInfo->upstreamNodeEpset = *pEpset;
-  pReadyInfo->nodeId = req.upstreamNodeId;
+  pReadyInfo->upstreamNodeId = upstreamNodeId;
   pReadyInfo->recvTs = taosGetTimestampMs();
-  pReadyInfo->checkpointId = req.checkpointId;
+  pReadyInfo->checkpointId = checkpointId;
+  pReadyInfo->childId = childId;
 
-  initRpcMsg(&pReadyInfo->msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead));
   return TSDB_CODE_SUCCESS;
 }
 
@@ -872,7 +973,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
   SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId);
 
   STaskCheckpointReadyInfo info = {0};
-  initCheckpointReadyInfo(&info, pTask, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId);
+  initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId);
 
   stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64
           "-0x%x (vgId:%d) idx:%d",
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index ae1d86e317..f6449829a3 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -1121,6 +1121,7 @@ void metaHbToMnode(void* param, void* tmrId) {
 
   SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid);
   if (pMeta == NULL) {
+    stError("invalid rid:%" PRId64 " failed to acquired stream-meta", rid);
     return;
   }
 
@@ -1341,8 +1342,8 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader)
   streamMetaWUnLock(pMeta);
 
   if (isLeader) {
-    stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId,
-           prevStage, stage, isLeader);
+    stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb, rid:%" PRId64,
+           pMeta->vgId, prevStage, stage, isLeader, pMeta->rid);
     streamMetaStartHb(pMeta);
   } else {
     stInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId,
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index f6524a69ab..7fb45a884c 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -999,9 +999,9 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
   pInfo->pReadyMsgList = taosArrayDestroy(pInfo->pReadyMsgList);
   pInfo->pCheckpointReadyRecvList = taosArrayDestroy(pInfo->pCheckpointReadyRecvList);
 
-  if (pInfo->pCheckTmr != NULL) {
-    taosTmrStop(pInfo->pCheckTmr);
-    pInfo->pCheckTmr = NULL;
+  if (pInfo->pChkptTriggerTmr != NULL) {
+    taosTmrStop(pInfo->pChkptTriggerTmr);
+    pInfo->pChkptTriggerTmr = NULL;
   }
 
   taosMemoryFree(pInfo);
@@ -1014,7 +1014,6 @@ void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) {
   pInfo->allUpstreamTriggerRecv = 0;
   pInfo->dispatchTrigger = false;
 
-  taosArrayClear(pInfo->pReadyMsgList);
   taosArrayClear(pInfo->pDispatchTriggerList);
   taosArrayClear(pInfo->pCheckpointReadyRecvList);
 }
\ No newline at end of file

From 5217131f500ccb65d2ca36150ce1904d91e536ad Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 3 Jun 2024 16:48:44 +0800
Subject: [PATCH 116/132] fix:[TD-30306]error in converity scan

---
 source/client/src/clientTmq.c | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 7bc6a2f2e6..13fc2ce16a 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1344,12 +1344,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     goto FAIL;
   }
 
-  if(pMsg->pData == NULL){
-    tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId);
-    code = TSDB_CODE_TSC_INTERNAL_ERROR;
-    goto FAIL;
-  }
-
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0);
   if (pRspWrapper == NULL) {
     tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId);
@@ -1362,6 +1356,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     goto END;
   }
 
+  if(pMsg->pData == NULL){
+    tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId);
+    code = TSDB_CODE_TSC_INTERNAL_ERROR;
+    goto END;
+  }
+
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t clientEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < clientEpoch) {
@@ -1388,7 +1388,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
       tDecoderClear(&decoder);
       taosReleaseRef(tmqMgmt.rsetId, refId);
       code = TSDB_CODE_OUT_OF_MEMORY;
-      goto FAIL;
+      goto END;
     }
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
@@ -1404,7 +1404,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
       tDecoderClear(&decoder);
       taosReleaseRef(tmqMgmt.rsetId, refId);
       code = TSDB_CODE_OUT_OF_MEMORY;
-      goto FAIL;
+      goto END;
     }
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead));
@@ -1415,7 +1415,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
       tDecoderClear(&decoder);
       taosReleaseRef(tmqMgmt.rsetId, refId);
       code = TSDB_CODE_OUT_OF_MEMORY;
-      goto FAIL;
+      goto END;
     }
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead));
@@ -1889,8 +1889,6 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
         terrno = pRspWrapper->code;
         tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId,
                  tstrerror(pRspWrapper->code));
-        taosFreeQitem(pRspWrapper);
-        return NULL;
       } else {
         if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {  // for vnode transform
           askEp(tmq, NULL, false, true);

From 111e1b2f60b5d22b6fd5107a067668e5dff622b9 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 3 Jun 2024 09:54:28 +0800
Subject: [PATCH 117/132] fix(stream): disable stream checkpoint when related
 fill-history task exists.

---
 source/dnode/mnode/impl/src/mndStream.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 48549fce42..79a04a5348 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1130,6 +1130,15 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) {
       ready = false;
       break;
     }
+
+    if (pEntry->hTaskId != 0) {
+      mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s related fill-history task:0x%" PRIx64
+             " exists, checkpoint not issued",
+             pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status),
+             pEntry->hTaskId);
+      ready = false;
+      break;
+    }
   }
 
   taosThreadMutexUnlock(&execInfo.lock);

From 3e38c1f9730df7ae886301562bb90e94847ee59f Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Mon, 3 Jun 2024 19:01:24 +0800
Subject: [PATCH 118/132] coverity scan for

---
 source/libs/executor/src/countwindowoperator.c    |  7 ++++++-
 source/libs/executor/src/scanoperator.c           | 15 +++++++++++++--
 .../libs/executor/src/streameventwindowoperator.c |  2 +-
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c
index 1f38264644..47b24421c8 100644
--- a/source/libs/executor/src/countwindowoperator.c
+++ b/source/libs/executor/src/countwindowoperator.c
@@ -67,7 +67,12 @@ static void clearWinStateBuff(SCountWindowResult* pBuff) {
 
 static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) {
   SCountWindowResult* pBuffInfo = taosArrayGet(pCountSup->pWinStates, pCountSup->stateIndex);
-  pCountSup->stateIndex = (pCountSup->stateIndex + 1) % taosArrayGetSize(pCountSup->pWinStates);
+  int32_t size = taosArrayGetSize(pCountSup->pWinStates);
+  // coverity scan
+  ASSERTS(size > 0, "WinStates is empty");
+  if (size > 0) {
+    pCountSup->stateIndex = (pCountSup->stateIndex + 1) % size;
+  }
   return pBuffInfo;
 }
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 851d2f2735..aabf410242 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1363,6 +1363,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
 }
 
 bool comparePrimaryKey(SColumnInfoData* pCol, int32_t rowId, void* pVal) {
+  // coverity scan
+  ASSERTS(pVal != NULL, "pVal should not be NULL");
+  if (!pVal) {
+    qError("failed to compare primary key, since primary key is null");
+    return false;
+  }
   void* pData = colDataGetData(pCol, rowId);
   if (IS_VAR_DATA_TYPE(pCol->info.type)) {
     int32_t colLen = varDataLen(pData);
@@ -1469,8 +1475,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
   }
 
   STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info;
-  qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion);
-  resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion);
+  // coverity scan
+  ASSERTS(pInfo->pUpdateInfo != NULL, "Failed to set data version, since pInfo->pUpdateInfo is NULL");
+  if (pInfo->pUpdateInfo) {
+    qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey,
+           pInfo->pUpdateInfo->maxDataVersion);
+    resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion);
+  }
   pInfo->pTableScanOp->status = OP_OPENED;
   return true;
 }
diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c
index 64d6244fe7..1809d0cc67 100644
--- a/source/libs/executor/src/streameventwindowoperator.c
+++ b/source/libs/executor/src/streameventwindowoperator.c
@@ -364,7 +364,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
     }
 
     if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
-      code = saveResult(curWin.winInfo, pSeUpdated);
+      saveResult(curWin.winInfo, pSeUpdated);
     }
 
     if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

From 91375c8ca15ce87ffaa2fd369d46df2f57a163d2 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Mon, 3 Jun 2024 19:13:14 +0800
Subject: [PATCH 119/132] fix: taosc exit issue

---
 include/libs/nodes/nodes.h             |  2 ++
 source/client/src/clientEnv.c          | 43 +++++++++++++++-----------
 source/libs/nodes/src/nodesUtilFuncs.c | 20 ++++++++++++
 3 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h
index 31091928c9..d0bf2e2dd4 100644
--- a/include/libs/nodes/nodes.h
+++ b/include/libs/nodes/nodes.h
@@ -104,6 +104,8 @@ typedef struct SNodeAllocator SNodeAllocator;
 int32_t nodesInitAllocatorSet();
 void    nodesDestroyAllocatorSet();
 int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId);
+int32_t nodesSimAcquireAllocator(int64_t allocatorId);
+int32_t nodesSimReleaseAllocator(int64_t allocatorId);
 int32_t nodesAcquireAllocator(int64_t allocatorId);
 int32_t nodesReleaseAllocator(int64_t allocatorId);
 int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId);
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 439103e5c4..6406bcbcd8 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -89,24 +89,28 @@ static void deregisterRequest(SRequestObj *pRequest) {
            "current:%d, app current:%d",
            pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
 
-  if (pRequest->pQuery && pRequest->pQuery->pRoot) {
-    if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type &&
-        (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) {
-      tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64
-               "us, planCost:%" PRId64 "us, exec:%" PRId64 "us",
-               duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs,
-               pRequest->metric.planCostUs, pRequest->metric.execCostUs);
-      atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
-      reqType = SLOW_LOG_TYPE_INSERT;
-    } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
-      tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64
-               "us, planCost:%" PRId64 "us, exec:%" PRId64 "us",
-               duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs,
-               pRequest->metric.planCostUs, pRequest->metric.execCostUs);
+  if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) {
+    if (pRequest->pQuery && pRequest->pQuery->pRoot) {
+      if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type &&
+          (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) {
+        tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64
+                 "us, planCost:%" PRId64 "us, exec:%" PRId64 "us",
+                 duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs,
+                 pRequest->metric.planCostUs, pRequest->metric.execCostUs);
+        atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
+        reqType = SLOW_LOG_TYPE_INSERT;
+      } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
+        tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64
+                 "us, planCost:%" PRId64 "us, exec:%" PRId64 "us",
+                 duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs,
+                 pRequest->metric.planCostUs, pRequest->metric.execCostUs);
 
-      atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
-      reqType = SLOW_LOG_TYPE_QUERY;
-    } 
+        atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
+        reqType = SLOW_LOG_TYPE_QUERY;
+      } 
+    }
+
+    nodesSimReleaseAllocator(pRequest->allocatorRefId);
   }
 
   if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
@@ -491,7 +495,10 @@ void doDestroyRequest(void *p) {
   }
   taosMemoryFree(pRequest->body.interParam);
 
-  qDestroyQuery(pRequest->pQuery);
+  if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) {
+    qDestroyQuery(pRequest->pQuery);
+    nodesSimReleaseAllocator(pRequest->allocatorRefId);
+  }
   nodesDestroyAllocator(pRequest->allocatorRefId);
 
   taosMemoryFreeClear(pRequest->effectiveUser);
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index cbffcde875..beedffc4f2 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -240,6 +240,26 @@ int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAlloc
   return code;
 }
 
+int32_t nodesSimAcquireAllocator(int64_t allocatorId) {
+  if (allocatorId <= 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId);
+  if (NULL == pAllocator) {
+    return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t nodesSimReleaseAllocator(int64_t allocatorId) {
+  if (allocatorId <= 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  return taosReleaseRef(g_allocatorReqRefPool, allocatorId);
+}
+
 int32_t nodesAcquireAllocator(int64_t allocatorId) {
   if (allocatorId <= 0) {
     return TSDB_CODE_SUCCESS;

From 407f79cfa69c08aec0822b3ab4cb8520b4f68a90 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 3 Jun 2024 19:14:45 +0800
Subject: [PATCH 120/132] fix(stream): fix dead-lock.

---
 source/libs/stream/src/streamCheckpoint.c | 24 +++++++++++------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index b0c4884c73..94d2198e31 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -332,17 +332,15 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
   // 1. not in checkpoint status now
   SStreamTaskState* pStat = streamTaskGetStatus(pTask);
   if (pStat->state != TASK_STATUS__CK) {
+    stError("s-task:%s status:%s discard checkpoint-ready msg from task:0x%x", id, pStat->name, downstreamTaskId);
     return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
   }
 
-  // 2. expired checkpoint-ready msg
-  if (pTask->chkInfo.checkpointId > checkpointId) {
-    // discard it directly
-    return -1;
-  }
-
-  // invalid checkpoint-ready msg
-  if (pInfo->activeId != checkpointId) {
+  // 2. expired checkpoint-ready msg, invalid checkpoint-ready msg
+  if (pTask->chkInfo.checkpointId > checkpointId || pInfo->activeId != checkpointId) {
+    stError("s-task:%s status:%s checkpointId:%" PRId64 " new arrival checkpoint-ready msg (checkpointId:%" PRId64
+            ") from task:0x%x, expired and discard ",
+            id, pStat->name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId);
     return -1;
   }
 
@@ -372,13 +370,14 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
   }
 
   int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
+  int32_t transId = pInfo->transId;
+  taosThreadMutexUnlock(&pInfo->lock);
+
   if (notReady == 0) {
-    stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task",
-            id);
-    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pInfo->activeId, pInfo->transId);
+    stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task", id);
+    appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId);
   }
 
-  taosThreadMutexUnlock(&pInfo->lock);
   return 0;
 }
 
@@ -944,7 +943,6 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
   SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
 
   int32_t taskId = 0;
-
   taosThreadMutexLock(&pInfo->lock);
 
   for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {

From ca15176f75e0af89149f65b7a890a0706249b99f Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 3 Jun 2024 19:23:55 +0800
Subject: [PATCH 121/132] docs:[TD-30027]modify subtable rules in stream
 document

---
 docs/en/12-taos-sql/14-stream.md | 2 +-
 docs/zh/12-taos-sql/14-stream.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md
index 3f93f69f0f..e7cefc1d7a 100644
--- a/docs/en/12-taos-sql/14-stream.md
+++ b/docs/en/12-taos-sql/14-stream.md
@@ -80,7 +80,7 @@ If a stream is created with PARTITION BY clause and SUBTABLE clause, the name of
 CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m);
 ```
 
-IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name(Starting from 3.2.3.0, in order to avoid the expression in subtable being unable to distinguish between different subtables, add '_groupId' to the end of subtable name).
+IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name(Starting from 3.2.3.0, in order to avoid the expression in subtable being unable to distinguish between different subtables, add '_stableName_groupId' to the end of subtable name).
 
 If the output length exceeds the limitation of TDengine(192), the name will be truncated. If the generated name is occupied by some other table, the creation and writing of the new subtable will be failed.
 
diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index c94df0a7ba..3d48116ec8 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -91,7 +91,7 @@ SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WIN
 CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m);
 ```
 
-PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名(从3.2.3.0开始，为了避免 sutable 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 _groupId)。
+PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名(从3.2.3.0开始，为了避免 sutable 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 __stableName_groupId)。
 
 注意，子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。
 

From 5ab81da4a0abcedd640d0213b681156383951b7a Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 4 Jun 2024 09:50:26 +0800
Subject: [PATCH 122/132] set scan mode for stream

---
 source/libs/executor/src/scanoperator.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 851d2f2735..3af738c63c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1314,6 +1314,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6
   pTableScanInfo->tableEndIndex = -1;
   pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
   pTableScanInfo->base.dataReader = NULL;
+  pTableScanInfo->scanMode = TABLE_SCAN__BLOCK_ORDER;
 }
 
 static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,

From 4fc21cc8b0f5fce72bf540b51c9106353f157f16 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 4 Jun 2024 10:08:18 +0800
Subject: [PATCH 123/132] refactor: do some internal refactor.

---
 include/libs/stream/tstream.h           |  3 ++-
 source/common/src/tglobal.c             |  2 +-
 source/dnode/mnode/impl/src/mndMain.c   |  2 +-
 source/dnode/mnode/impl/src/mndStream.c |  2 +-
 source/libs/stream/src/streamDispatch.c |  8 ++++----
 source/libs/stream/src/streamTask.c     | 11 ++++++++---
 6 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 26c6631ee4..d07a302920 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -319,7 +319,8 @@ typedef struct SDispatchMsgInfo {
   int32_t             retryCount;  // retry send data count
   int64_t             startTs;     // dispatch start time, record total elapsed time for dispatch
   SArray*             pRetryList;  // current dispatch successfully completed node of downstream
-  void*               pTimer;      // used to dispatch data after a given time duration
+  void*               pRetryTmr;   // used to dispatch data after a given time duration
+  void*               pRspTmr;      // used to dispatch data after a given time duration
 } SDispatchMsgInfo;
 
 typedef struct STaskQueue {
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 2fefeb4cf2..f034244c69 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -273,7 +273,7 @@ int32_t tsCompactPullupInterval = 10;
 int32_t tsMqRebalanceInterval = 2;
 int32_t tsStreamCheckpointInterval = 60;
 float   tsSinkDataRate = 2.0;
-int32_t tsStreamNodeCheckInterval = 16;
+int32_t tsStreamNodeCheckInterval = 20;
 int32_t tsMaxConcurrentCheckpoint = 1;
 int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushIntervalSec = 10;
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index c82007fb59..cad8c6d745 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -345,7 +345,7 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
     mndCalMqRebalance(pMnode);
   }
 
-  if (sec % 30 == 0) {  // send the checkpoint info every 10 sec
+  if (sec % 30 == 0) {  // send the checkpoint info every 30 sec
     mndStreamCheckpointTimer(pMnode);
   }
 
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index db4b345536..e108ba557a 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1062,7 +1062,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
   bool    allReady = true;
   SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
   if (!allReady) {
-    mWarn("not all vnodes ready");
+    mWarn("not all vnodes ready, quit from vnodes status check");
     taosArrayDestroy(pNodeSnapshot);
     taosThreadMutexUnlock(&execInfo.lock);
     return 0;
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 42a4e4e8fb..fb5f1e33c5 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -420,7 +420,7 @@ static void doRetryDispatchData(void* param, void* tmrId) {
         }
       }
 
-      stDebug("s-task:%s complete re-try shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr,
+      stDebug("s-task:%s complete retry shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr,
               numOfFailed, msgId);
     } else {
       int32_t vgId = pTask->outputInfo.fixedDispatcher.nodeId;
@@ -461,10 +461,10 @@ void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration) {
   stTrace("s-task:%s retry send dispatch data in %" PRId64 "ms, in timer msgId:%d, retryTimes:%d", pTask->id.idStr,
          waitDuration, pTask->execInfo.dispatch, pTask->msgInfo.retryCount);
 
-  if (pTask->msgInfo.pTimer != NULL) {
-    taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pTimer);
+  if (pTask->msgInfo.pRetryTmr != NULL) {
+    taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr);
   } else {
-    pTask->msgInfo.pTimer = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamTimer);
+    pTask->msgInfo.pRetryTmr = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamTimer);
   }
 }
 
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 7fb45a884c..834daf15d0 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -228,9 +228,9 @@ void tFreeStreamTask(SStreamTask* pTask) {
     pTask->hTaskInfo.pTimer = NULL;
   }
 
-  if (pTask->msgInfo.pTimer != NULL) {
-    /*bool ret = */taosTmrStop(pTask->msgInfo.pTimer);
-    pTask->msgInfo.pTimer = NULL;
+  if (pTask->msgInfo.pRetryTmr != NULL) {
+    /*bool ret = */taosTmrStop(pTask->msgInfo.pRetryTmr);
+    pTask->msgInfo.pRetryTmr = NULL;
   }
 
   if (pTask->inputq.queue) {
@@ -1004,6 +1004,11 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
     pInfo->pChkptTriggerTmr = NULL;
   }
 
+  if (pInfo->pSendReadyMsgTmr != NULL) {
+    taosTmrStop(pInfo->pSendReadyMsgTmr);
+    pInfo->pSendReadyMsgTmr = NULL;
+  }
+
   taosMemoryFree(pInfo);
 }
 

From e1b32e12a14ad16f318c6701157cdce04f4b469c Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Tue, 4 Jun 2024 05:07:56 +0000
Subject: [PATCH 124/132] TD-30422

---
 source/dnode/mgmt/exe/dmMain.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c
index 76b0565402..6fdd1ba33d 100644
--- a/source/dnode/mgmt/exe/dmMain.c
+++ b/source/dnode/mgmt/exe/dmMain.c
@@ -403,13 +403,6 @@ int mainWindows(int argc, char **argv) {
     return -1;
   }
 
-  if(dmGetEncryptKey() != 0){
-    dError("failed to start since failed to get encrypt key");
-    taosCloseLog();
-    taosCleanupArgs();
-    return -1;
-  };
-
   if (taosConvInit() != 0) {
     dError("failed to init conv");
     taosCloseLog();
@@ -447,6 +440,13 @@ int mainWindows(int argc, char **argv) {
   osSetProcPath(argc, (char **)argv);
   taosCleanupArgs();
 
+  if(dmGetEncryptKey() != 0){
+    dError("failed to start since failed to get encrypt key");
+    taosCloseLog();
+    taosCleanupArgs();
+    return -1;
+  };
+
   if (dmInit() != 0) {
     if (terrno == TSDB_CODE_NOT_FOUND) {
       dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support");

From 6f696a3933d83a15704f969234ff06c30dc6855e Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 4 Jun 2024 16:27:51 +0800
Subject: [PATCH 125/132] fix: insert into select constant issue

---
 source/libs/scalar/src/scalar.c            |  4 ++++
 tests/script/tsim/insert/insert_select.sim | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index 5f43ae9f3c..50de5e760d 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -1213,6 +1213,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) {
 
   res->translate = true;
 
+  strcpy(res->node.aliasName, node->node.aliasName);
   res->node.resType.type = output.columnData->info.type;
   res->node.resType.bytes = output.columnData->info.bytes;
   res->node.resType.scale = output.columnData->info.scale;
@@ -1268,6 +1269,7 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) {
   res->node.resType = node->node.resType;
   res->translate = true;
 
+  strcpy(res->node.aliasName, node->node.aliasName);
   int32_t type = output.columnData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
     res->datum.p = output.columnData->pData;
@@ -1309,6 +1311,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
 
   res->translate = true;
 
+  strcpy(res->node.aliasName, node->node.aliasName);
   res->node.resType = node->node.resType;
   if (colDataIsNull_s(output.columnData, 0)) {
     res->isNull = true;
@@ -1364,6 +1367,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) {
 
   res->translate = true;
 
+  strcpy(res->node.aliasName, node->node.aliasName);
   res->node.resType = node->node.resType;
   if (colDataIsNull_s(output.columnData, 0)) {
     res->isNull = true;
diff --git a/tests/script/tsim/insert/insert_select.sim b/tests/script/tsim/insert/insert_select.sim
index 333964b1d6..888f349fbb 100644
--- a/tests/script/tsim/insert/insert_select.sim
+++ b/tests/script/tsim/insert/insert_select.sim
@@ -61,5 +61,23 @@ if $data02 != 1 then
   return -1
 endi
 
+sql insert into t2 (ts, b, a) select ts + 1, 11, 12 from t1;
+sql select * from t2;
+if $rows != 2 then
+  return -1
+endi
+if $data01 != 2 then
+  return -1
+endi
+if $data02 != 1 then
+  return -1
+endi
+if $data11 != 12 then
+  return -1
+endi
+if $data12 != 11 then
+  return -1
+endi
+
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 1f73bb106e759992a39851abbfa00b4c759fd476 Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Tue, 4 Jun 2024 17:13:48 +0800
Subject: [PATCH 126/132] enh: add more log for restore

---
 source/dnode/vnode/src/vnd/vnodeCommit.c |  6 ++++++
 source/libs/sync/src/syncPipeline.c      | 11 +++++++++++
 2 files changed, 17 insertions(+)

diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index f071775990..5acaf2bce4 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -160,6 +160,12 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
                  (atExit && (pVnode->inUse->size > 0 || pVnode->pMeta->changed ||
                              pVnode->state.applied - pVnode->state.committed > 4096));
   }
+  vTrace("vgId:%d, should commit:%d, disk available:%d, buffer size:%" PRId64 ", node size:%" PRId64
+         ", meta changed:%d"
+         ", state:[%" PRId64 ",%" PRId64 "]",
+         TD_VID(pVnode), needCommit, diskAvail, pVnode->inUse ? pVnode->inUse->size : 0,
+         pVnode->inUse ? pVnode->inUse->node.size : 0, pVnode->pMeta->changed, pVnode->state.applied,
+         pVnode->state.committed);
   taosThreadMutexUnlock(&pVnode->mutex);
   return needCommit;
 }
diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c
index 3543ed574c..796a45d997 100644
--- a/source/libs/sync/src/syncPipeline.c
+++ b/source/libs/sync/src/syncPipeline.c
@@ -874,6 +874,17 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
       sError("vgId:%d, failed to get prev log term since %s. index:%" PRId64, pNode->vgId, terrstr(), index + 1);
       return -1;
     }
+
+    if (pMsg->matchIndex == -1) {
+      // first time to restore
+      sInfo("vgId:%d, first time to restore sync log repl. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64
+            ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 "), index:%" PRId64
+            ", firstVer:%" PRId64 ", term:%" PRId64 ", lastMatchTerm:%" PRId64,
+            pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
+            pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex, index, firstVer, term,
+            pMsg->lastMatchTerm);
+    }
+
     if ((index + 1 < firstVer) || (term < 0) ||
         (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) {
       ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);

From 29648be30ded486ccba66b18df204220a48fcaca Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 4 Jun 2024 23:28:08 +0800
Subject: [PATCH 127/132] fix(stream): add the new node info when adding stream
 tasks.

---
 source/dnode/mnode/impl/inc/mndStream.h     |  1 +
 source/dnode/mnode/impl/src/mndStream.c     | 89 ++++++++++++---------
 source/dnode/mnode/impl/src/mndStreamHb.c   | 11 ++-
 source/dnode/mnode/impl/src/mndStreamUtil.c | 24 ++++++
 4 files changed, 83 insertions(+), 42 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index 2800aecdfa..6d2a89ddc9 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -120,6 +120,7 @@ void             destroyStreamTaskIter(SStreamTaskIter *pIter);
 bool             streamTaskIterNextTask(SStreamTaskIter *pIter);
 SStreamTask     *streamTaskIterGetCurrent(SStreamTaskIter *pIter);
 void             mndInitExecInfo();
+void             removeExpiredNodeInfo(const SArray *pNodeSnapshot);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index e108ba557a..bbf2ad63ce 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -62,7 +62,7 @@ static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq);
 static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
 
 static void     removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
-static int32_t  removeExpirednodeEntryAndTask(SArray *pNodeSnapshot);
+static int32_t  removeExpiredNodeEntryAndTask(SArray *pNodeSnapshot);
 static int32_t  doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len);
 static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
 
@@ -692,6 +692,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
   SStreamObj  streamObj = {0};
   char       *sql = NULL;
   int32_t     sqlLen = 0;
+  const char* pMsg = "create stream tasks on dnodes";
+
   terrno = TSDB_CODE_SUCCESS;
 
   SCMCreateStreamReq createReq = {0};
@@ -704,8 +706,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
   terrno = TSDB_CODE_MND_INVALID_PLATFORM;
   goto _OVER;
 #endif
-  mInfo("stream:%s, start to create stream, sql:%s", createReq.name, createReq.sql);
 
+  mInfo("stream:%s, start to create stream, sql:%s", createReq.name, createReq.sql);
   if (mndCheckCreateStreamReq(&createReq) != 0) {
     mError("stream:%s, failed to create since %s", createReq.name, terrstr());
     goto _OVER;
@@ -745,8 +747,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  STrans *pTrans =
-      doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, "create stream tasks on dnodes");
+  STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg);
   if (pTrans == NULL) {
     goto _OVER;
   }
@@ -789,7 +790,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
   // add into buffer firstly
   // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already.
   taosThreadMutexLock(&execInfo.lock);
-  mDebug("stream stream:%s start to register tasks into task_node_list", createReq.name);
+  mDebug("stream stream:%s start to register tasks into task nodeList", createReq.name);
   saveStreamTasksInfo(&streamObj, &execInfo);
   taosThreadMutexUnlock(&execInfo.lock);
 
@@ -1030,7 +1031,7 @@ _ERR:
 }
 
 int32_t initStreamNodeList(SMnode *pMnode) {
-  if (execInfo.pNodeList == NULL || (taosArrayGetSize(execInfo.pNodeList) == 0)) {
+  if (taosArrayGetSize(execInfo.pNodeList) == 0) {
     execInfo.pNodeList = taosArrayDestroy(execInfo.pNodeList);
     execInfo.pNodeList = extractNodeListFromStream(pMnode);
   }
@@ -2203,8 +2204,8 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
     epsetToStr(&pEntry->epset, buf, tListLen(buf));
     mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf);
   }
-  taosHashCleanup(pHash);
 
+  taosHashCleanup(pHash);
   return plist;
 }
 
@@ -2242,15 +2243,17 @@ static bool taskNodeExists(SArray *pList, int32_t nodeId) {
   return false;
 }
 
-int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
+int32_t removeExpiredNodeEntryAndTask(SArray *pNodeSnapshot) {
   SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
 
   int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
   for (int32_t i = 0; i < numOfTask; ++i) {
-    STaskId          *pId = taosArrayGet(execInfo.pTaskList, i);
-    STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
+    STaskId *pId = taosArrayGet(execInfo.pTaskList, i);
 
-    if (pEntry->nodeId == SNODE_HANDLE) continue;
+    STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
+    if (pEntry->nodeId == SNODE_HANDLE) {
+      continue;
+    }
 
     bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
     if (!existed) {
@@ -2266,24 +2269,8 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
   mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
          (int32_t)taosArrayGetSize(execInfo.pTaskList));
 
-  int32_t size = taosArrayGetSize(pNodeSnapshot);
-  SArray *pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry));
-  for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) {
-    SNodeEntry *p = taosArrayGet(execInfo.pNodeList, i);
+  removeExpiredNodeInfo(pNodeSnapshot);
 
-    for (int32_t j = 0; j < size; ++j) {
-      SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j);
-      if (pEntry->nodeId == p->nodeId) {
-        taosArrayPush(pValidNodeEntryList, p);
-        break;
-      }
-    }
-  }
-
-  taosArrayDestroy(execInfo.pNodeList);
-  execInfo.pNodeList = pValidNodeEntryList;
-
-  mDebug("remain %d valid node entries", (int32_t)taosArrayGetSize(pValidNodeEntryList));
   taosArrayDestroy(pRemovedTasks);
   return 0;
 }
@@ -2314,9 +2301,9 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
     return 0;
   }
 
-  bool    allVgroupsReady = true;
-  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVgroupsReady);
-  if (!allVgroupsReady) {
+  bool    allReady = true;
+  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
+  if (!allReady) {
     taosArrayDestroy(pNodeSnapshot);
     atomic_store_32(&mndNodeCheckSentinel, 0);
     mWarn("not all vnodes are ready, ignore the exec nodeUpdate check");
@@ -2324,31 +2311,30 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
   }
 
   taosThreadMutexLock(&execInfo.lock);
-  removeExpirednodeEntryAndTask(pNodeSnapshot);
+  removeExpiredNodeEntryAndTask(pNodeSnapshot);
 
   SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot);
   if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) {
     // kill current active checkpoint transaction, since the transaction is vnode wide.
     killAllCheckpointTrans(pMnode, &changeInfo);
-
     code = mndProcessVgroupChange(pMnode, &changeInfo);
 
     // keep the new vnode snapshot if success
     if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) {
-      mDebug("create trans successfully, update cached node list");
       taosArrayDestroy(execInfo.pNodeList);
-      execInfo.pNodeList = pNodeSnapshot;
+      execInfo.pNodeList = extractNodeListFromStream(pMnode);
       execInfo.ts = ts;
+      mDebug("create trans successfully, update cached node list, numOfNodes:%d", taosArrayGetSize(execInfo.pNodeList));
     } else {
       mError("unexpected code during create nodeUpdate trans, code:%s", tstrerror(code));
-      taosArrayDestroy(pNodeSnapshot);
     }
   } else {
     mDebug("no update found in nodeList");
-    taosArrayDestroy(pNodeSnapshot);
   }
 
+  taosArrayDestroy(pNodeSnapshot);
   taosThreadMutexUnlock(&execInfo.lock);
+
   taosArrayDestroy(changeInfo.pUpdateNodeList);
   taosHashCleanup(changeInfo.pDBMap);
 
@@ -2385,8 +2371,27 @@ void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
 
       taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry));
       taosArrayPush(pExecNode->pTaskList, &id);
-      mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId,
-            (int32_t)taosArrayGetSize(pExecNode->pTaskList));
+
+      int32_t num = (int32_t)taosArrayGetSize(pExecNode->pTaskList);
+      mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId, num);
+
+      // add the new vgroups if not added yet
+      bool exist = false;
+      for(int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) {
+        SNodeEntry* pEntry = taosArrayGet(pExecNode->pNodeList, j);
+        if (pEntry->nodeId == pTask->info.nodeId) {
+          exist = true;
+          break;
+        }
+      }
+
+      if (!exist) {
+        SNodeEntry nodeEntry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId};
+        epsetAssign(&nodeEntry.epset, &pTask->info.epSet);
+
+        taosArrayPush(pExecNode->pNodeList, &nodeEntry);
+        mInfo("vgId:%d added into nodeList, total:%d", nodeEntry.nodeId, (int)taosArrayGetSize(pExecNode->pNodeList));
+      }
     }
   }
 
@@ -2394,6 +2399,8 @@ void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
 }
 
 void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
+  taosThreadMutexLock(&pExecNode->lock);
+
   SStreamTaskIter *pIter = createStreamTaskIter(pStream);
   while (streamTaskIterNextTask(pIter)) {
     SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
@@ -2416,8 +2423,10 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
     }
   }
 
-  destroyStreamTaskIter(pIter);
   ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
+  taosThreadMutexUnlock(&pExecNode->lock);
+
+  destroyStreamTaskIter(pIter);
 }
 
 static void doAddTaskId(SArray *pList, int32_t taskId, int64_t uid, int32_t numOfTotal) {
diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c
index 9bd7b3b18f..778fd295f7 100644
--- a/source/dnode/mnode/impl/src/mndStreamHb.c
+++ b/source/dnode/mnode/impl/src/mndStreamHb.c
@@ -131,18 +131,26 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) {
     int32_t *pVgId = taosArrayGet(pNodeList, k);
     mInfo("set node expired for nodeId:%d, total:%d", *pVgId, num);
 
+    bool    setFlag = false;
     int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList);
+
     for (int i = 0; i < numOfNodes; ++i) {
       SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i);
 
       if (pNodeEntry->nodeId == *pVgId) {
         mInfo("vgId:%d expired for some stream tasks, needs update nodeEp", *pVgId);
         pNodeEntry->stageUpdated = true;
+        setFlag = true;
         break;
       }
     }
-  }
 
+    if (!setFlag) {
+      mError("failed to set nodeUpdate flag, nodeId:%d not exists in nodelist, update it", *pVgId);
+      ASSERT(0);
+      return TSDB_CODE_FAILED;
+    }
+  }
   return TSDB_CODE_SUCCESS;
 }
 
@@ -361,7 +369,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
     pHead->vgId = htonl(req.vgId);
 
     tmsgSendRsp(&rsp);
-
     pReq->info.handle = NULL;   // disable auto rsp
   }
 
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index e53908eeed..54279161ab 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -135,6 +135,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
     char buf[256] = {0};
     epsetToStr(&entry.epset, buf, tListLen(buf));
     mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
+
     taosArrayPush(pVgroupListSnapshot, &entry);
     sdbRelease(pSdb, pObj);
   }
@@ -571,6 +572,29 @@ void mndInitExecInfo() {
   execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK);
   execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK);
   execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK);
+  execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry));
 
   taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
 }
+
+void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
+  SArray *pValidList = taosArrayInit(4, sizeof(SNodeEntry));
+  int32_t size = taosArrayGetSize(pNodeSnapshot);
+
+  for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) {
+    SNodeEntry *p = taosArrayGet(execInfo.pNodeList, i);
+
+    for (int32_t j = 0; j < size; ++j) {
+      SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j);
+      if (pEntry->nodeId == p->nodeId) {
+        taosArrayPush(pValidList, p);
+        break;
+      }
+    }
+  }
+
+  taosArrayDestroy(execInfo.pNodeList);
+  execInfo.pNodeList = pValidList;
+
+  mDebug("remain %d valid node entries after clean expired nodes info", (int32_t)taosArrayGetSize(pValidList));
+}
\ No newline at end of file

From bf9a5135fb13b9333933ec848864e3d13ae2dfb1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 5 Jun 2024 00:13:57 +0800
Subject: [PATCH 128/132] fix(stream): disable timer for checkpoint-ready msg
 in rsma.

---
 source/libs/stream/src/streamDispatch.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index fb5f1e33c5..2e776313e0 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -769,16 +769,16 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
   stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num);
 
   // start to check if checkpoint ready msg has successfully received by upstream tasks.
-  pActiveInfo->pSendReadyMsgTmr = NULL;
+  if (pTask->info.taskLevel == TASK_LEVEL__SINK || pTask->info.taskLevel == TASK_LEVEL__AGG) {
+    int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref);
+    streamMetaAcquireOneTask(pTask);
 
-  int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
-  stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref);
-  streamMetaAcquireOneTask(pTask);
-
-  if (pActiveInfo->pSendReadyMsgTmr == NULL) {
-    pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer);
-  } else {
-    taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr);
+    if (pActiveInfo->pSendReadyMsgTmr == NULL) {
+      pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer);
+    } else {
+      taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr);
+    }
   }
 
   return TSDB_CODE_SUCCESS;

From 68915197ee04b509da58aadefe1ea85a1e2fcd83 Mon Sep 17 00:00:00 2001
From: charles <flyingangel_2008@163.com>
Date: Wed, 5 Jun 2024 08:25:39 +0800
Subject: [PATCH 129/132] update test cases to fix uncertainty results by
 charles

---
 tests/army/community/insert/insert_basic.py | 24 +++---
 tests/system-test/1-insert/insert_double.py | 90 ++++++++++-----------
 tests/system-test/2-query/td-28068.py       | 16 ++--
 3 files changed, 65 insertions(+), 65 deletions(-)

diff --git a/tests/army/community/insert/insert_basic.py b/tests/army/community/insert/insert_basic.py
index 481db4eadd..1f2488a756 100644
--- a/tests/army/community/insert/insert_basic.py
+++ b/tests/army/community/insert/insert_basic.py
@@ -35,18 +35,18 @@ class TDTestCase(TBase):
         tdSql.execute("create database db_geometry;")
         tdSql.execute("use db_geometry;")
         tdSql.execute("create table t_ge (ts timestamp, id int, c1 GEOMETRY(512));")
-        tdSql.execute("insert into t_ge values(now, 1, 'MULTIPOINT ((0 0), (1 1))');")
-        tdSql.execute("insert into t_ge values(now, 1, 'MULTIPOINT (0 0, 1 1)');")
-        tdSql.execute("insert into t_ge values(now, 2, 'POINT (0 0)');")
-        tdSql.execute("insert into t_ge values(now, 2, 'POINT EMPTY');")
-        tdSql.execute("insert into t_ge values(now, 3, 'LINESTRING (0 0, 0 1, 1 2)');")
-        tdSql.execute("insert into t_ge values(now, 3, 'LINESTRING EMPTY');")
-        tdSql.execute("insert into t_ge values(now, 4, 'POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))');")
-        tdSql.execute("insert into t_ge values(now, 4, 'POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))');")
-        tdSql.execute("insert into t_ge values(now, 4, 'POLYGON EMPTY');")
-        tdSql.execute("insert into t_ge values(now, 5, 'MULTILINESTRING ((0 0, 1 1), (2 2, 3 3))');")
-        tdSql.execute("insert into t_ge values(now, 6, 'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((4 3, 6 3, 6 1, 4 1, 4 3)))');")
-        tdSql.execute("insert into t_ge values(now, 7, 'GEOMETRYCOLLECTION (MULTIPOINT((0 0), (1 1)), POINT(3 4), LINESTRING(2 3, 3 4))');")
+        tdSql.execute("insert into t_ge values(1717122943000, 1, 'MULTIPOINT ((0 0), (1 1))');")
+        tdSql.execute("insert into t_ge values(1717122944000, 1, 'MULTIPOINT (0 0, 1 1)');")
+        tdSql.execute("insert into t_ge values(1717122945000, 2, 'POINT (0 0)');")
+        tdSql.execute("insert into t_ge values(1717122946000, 2, 'POINT EMPTY');")
+        tdSql.execute("insert into t_ge values(1717122947000, 3, 'LINESTRING (0 0, 0 1, 1 2)');")
+        tdSql.execute("insert into t_ge values(1717122948000, 3, 'LINESTRING EMPTY');")
+        tdSql.execute("insert into t_ge values(1717122949000, 4, 'POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))');")
+        tdSql.execute("insert into t_ge values(1717122950000, 4, 'POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))');")
+        tdSql.execute("insert into t_ge values(1717122951000, 4, 'POLYGON EMPTY');")
+        tdSql.execute("insert into t_ge values(1717122952000, 5, 'MULTILINESTRING ((0 0, 1 1), (2 2, 3 3))');")
+        tdSql.execute("insert into t_ge values(1717122953000, 6, 'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((4 3, 6 3, 6 1, 4 1, 4 3)))');")
+        tdSql.execute("insert into t_ge values(1717122954000, 7, 'GEOMETRYCOLLECTION (MULTIPOINT((0 0), (1 1)), POINT(3 4), LINESTRING(2 3, 3 4))');")
         tdSql.query("select * from t_ge;")
         tdSql.checkRows(12)
         tdSql.query("select * from t_ge where id=1;")
diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py
index b7af7237db..d6dada32cd 100644
--- a/tests/system-test/1-insert/insert_double.py
+++ b/tests/system-test/1-insert/insert_double.py
@@ -26,30 +26,30 @@ class TDTestCase:
         tdSql.execute(f"drop table if exists {table_name}")
         tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)")
 
-        tdSql.execute(f"insert into {table_name} values(now, -16, +6)")
-        tdSql.execute(f"insert into {table_name} values(now, 80.99, +0042)")
-        tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)")
-        tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)")
-        tdSql.execute(f"insert into {table_name} values(now, -12., +3.)")
-        tdSql.execute(f"insert into {table_name} values(now, -0.12, +3.0)")
-        tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)")
-        tdSql.execute(f"insert into {table_name} values(now, -2e1,  +2e2)")
-        tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)")
-        tdSql.execute(f"insert into {table_name} values(now, -0x40, +0b10000)")
-        tdSql.execute(f"insert into {table_name} values(now, -0b10000, +0x40)")
+        tdSql.execute(f"insert into {table_name} values(1717122943000, -16, +6)")
+        tdSql.execute(f"insert into {table_name} values(1717122944000, 80.99, +0042)")
+        tdSql.execute(f"insert into {table_name} values(1717122945000, -0042, +80.99)")
+        tdSql.execute(f"insert into {table_name} values(1717122946000, 52.34354, 18.6)")
+        tdSql.execute(f"insert into {table_name} values(1717122947000, -12., +3.)")
+        tdSql.execute(f"insert into {table_name} values(1717122948000, -0.12, +3.0)")
+        tdSql.execute(f"insert into {table_name} values(1717122949000, -2.3e1, +2.324e2)")
+        tdSql.execute(f"insert into {table_name} values(1717122950000, -2e1,  +2e2)")
+        tdSql.execute(f"insert into {table_name} values(1717122951000, -2.e1, +2.e2)")
+        tdSql.execute(f"insert into {table_name} values(1717122952000, -0x40, +0b10000)")
+        tdSql.execute(f"insert into {table_name} values(1717122953000, -0b10000, +0x40)")
 
         # str support
-        tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')")
-        tdSql.execute(f"insert into {table_name} values(now, ' -80.99', ' +0042')")
-        tdSql.execute(f"insert into {table_name} values(now, ' -0042', ' +80.99')")
-        tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')")
-        tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')")
-        tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')")
-        tdSql.execute(f"insert into {table_name} values(now, '-2.e1', '+2.e2')")
-        tdSql.execute(f"insert into {table_name} values(now, '-2e1',  '+2e2')")
-        tdSql.execute(f"insert into {table_name} values(now, '-2.3e1', '+2.324e2')")
-        tdSql.execute(f"insert into {table_name} values(now, '-0x40', '+0b10010')")
-        tdSql.execute(f"insert into {table_name} values(now, '-0b10010', '+0x40')")
+        tdSql.execute(f"insert into {table_name} values(1717122954000, '-16', '+6')")
+        tdSql.execute(f"insert into {table_name} values(1717122955000, ' -80.99', ' +0042')")
+        tdSql.execute(f"insert into {table_name} values(1717122956000, ' -0042', ' +80.99')")
+        tdSql.execute(f"insert into {table_name} values(1717122957000, '52.34354', '18.6')")
+        tdSql.execute(f"insert into {table_name} values(1717122958000, '-12.', '+5.')")
+        tdSql.execute(f"insert into {table_name} values(1717122959000, '-.12', '+.5')")
+        tdSql.execute(f"insert into {table_name} values(1717122960000, '-2.e1', '+2.e2')")
+        tdSql.execute(f"insert into {table_name} values(1717122961000, '-2e1',  '+2e2')")
+        tdSql.execute(f"insert into {table_name} values(1717122962000, '-2.3e1', '+2.324e2')")
+        tdSql.execute(f"insert into {table_name} values(1717122963000, '-0x40', '+0b10010')")
+        tdSql.execute(f"insert into {table_name} values(1717122964000, '-0b10010', '+0x40')")
 
         tdSql.query(f"select * from {table_name}")
         tdSql.checkRows(22)
@@ -64,22 +64,22 @@ class TDTestCase:
         min_u = 0
         print("val:", baseval, negval, posval, max_i)
 
-        tdSql.execute(f"insert into {table_name} values(now, {negval}, {posval})")
-        tdSql.execute(f"insert into {table_name} values(now, -{baseval}, {baseval})")
-        tdSql.execute(f"insert into {table_name} values(now, {max_i}, {max_u})")
-        tdSql.execute(f"insert into {table_name} values(now, {min_i}, {min_u})")
+        tdSql.execute(f"insert into {table_name} values(1717122965000, {negval}, {posval})")
+        tdSql.execute(f"insert into {table_name} values(1717122966000, -{baseval}, {baseval})")
+        tdSql.execute(f"insert into {table_name} values(1717122967000, {max_i}, {max_u})")
+        tdSql.execute(f"insert into {table_name} values(1717122968000, {min_i}, {min_u})")
 
         tdSql.query(f"select * from {table_name}")
         tdSql.checkRows(26)
         
         # error case
-        tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})")
-        tdSql.error(f"insert into {table_name} values(now, 0, -1)")
-        tdSql.error(f"insert into {table_name} values(now, 0, -2.0)")
-        tdSql.error(f"insert into {table_name} values(now, 0, '-2.0')")
-        tdSql.error(f"insert into {table_name} values(now, {max_i+1}, 0)")
-        tdSql.error(f"insert into {table_name} values(now, {min_i-1}, 0)")
-        tdSql.error(f"insert into {table_name} values(now, '{min_i-1}', 0)")
+        tdSql.error(f"insert into {table_name} values(1717122969000, 0, {max_u+1})")
+        tdSql.error(f"insert into {table_name} values(1717122970000, 0, -1)")
+        tdSql.error(f"insert into {table_name} values(1717122971000, 0, -2.0)")
+        tdSql.error(f"insert into {table_name} values(1717122972000, 0, '-2.0')")
+        tdSql.error(f"insert into {table_name} values(1717122973000, {max_i+1}, 0)")
+        tdSql.error(f"insert into {table_name} values(1717122974000, {min_i-1}, 0)")
+        tdSql.error(f"insert into {table_name} values(1717122975000, '{min_i-1}', 0)")
 
     def test_tags(self, stable_name, dtype, bits):
         tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})")
@@ -93,20 +93,20 @@ class TDTestCase:
         max_u = 2*bigval - 1
         min_u = 0
 
-        tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(now, {negval}, {posval})")
-        tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(now, -{baseval} , {baseval})")
-        tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(now, {max_i}, {max_u})")
-        tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(now, {min_i}, {min_u})")
+        tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(1717122976000, {negval}, {posval})")
+        tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(1717122977000, -{baseval} , {baseval})")
+        tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(1717122978000, {max_i}, {max_u})")
+        tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(1717122979000, {min_i}, {min_u})")
         
-        tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(now, '{negval}', '{posval}')")
-        tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(now, '-{baseval}' , '{baseval}')")
-        tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(now, '{max_i}', '{max_u}')")
-        tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(now, '{min_i}', '{min_u}')")
+        tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(1717122980000, '{negval}', '{posval}')")
+        tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(1717122981000, '-{baseval}' , '{baseval}')")
+        tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(1717122982000, '{max_i}', '{max_u}')")
+        tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(1717122983000, '{min_i}', '{min_u}')")
 
-        tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(now, {negval}, {posval})")
-        tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(now, -{baseval} , {baseval})")
-        tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(now, {max_i}, {max_u})")
-        tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(now, {min_i}, {min_u})")
+        tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(1717122984000, {negval}, {posval})")
+        tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(1717122985000, -{baseval} , {baseval})")
+        tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(1717122986000, {max_i}, {max_u})")
+        tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(1717122987000, {min_i}, {min_u})")
 
         tdSql.query(f"select * from {stable_name}")
         tdSql.checkRows(12)
diff --git a/tests/system-test/2-query/td-28068.py b/tests/system-test/2-query/td-28068.py
index 0dfaf8e126..0a7e75fef2 100644
--- a/tests/system-test/2-query/td-28068.py
+++ b/tests/system-test/2-query/td-28068.py
@@ -10,14 +10,14 @@ class TDTestCase:
         tdSql.execute("create database td_28068;")
         tdSql.execute("create database if not exists td_28068;")
         tdSql.execute("create stable td_28068.st (ts timestamp, test_case nchar(10), time_cost float, num float) tags (branch nchar(10), scenario nchar(10));")
-        tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (now(), 'query1', 1,2);")
-        tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (now(), 'query1', 2,3);")
-        tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (now(), 'query1', 10,1);")
-        tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (now(), 'query1', 11,5);")
-        tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (now(), 'query1', 20,4);")
-        tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (now(), 'query1', 30,1);")
-        tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (now(), 'query1', 8,8);")
-        tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (now(), 'query1', 9,10);")
+        tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (1717122943000, 'query1', 1,2);")
+        tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (1717122944000, 'query1', 2,3);")
+        tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (1717122945000, 'query1', 10,1);")
+        tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (1717122946000, 'query1', 11,5);")
+        tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (1717122947000, 'query1', 20,4);")
+        tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (1717122948000, 'query1', 30,1);")
+        tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (1717122949000, 'query1', 8,8);")
+        tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (1717122950000, 'query1', 9,10);")
 
     def run(self):
         tdSql.query('select last(ts) as ts, last(branch) as branch, last(scenario) as scenario, last(test_case) as test_case  from td_28068.st group by branch, scenario order by last(branch);')

From 82293406f131f6bbffa2ef1f90fb55fb4323c9a7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 5 Jun 2024 09:14:33 +0800
Subject: [PATCH 130/132] fix(stream): fix syntax error.

---
 source/dnode/mnode/impl/src/mndStream.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index bbf2ad63ce..9bfda65108 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -2324,7 +2324,8 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
       taosArrayDestroy(execInfo.pNodeList);
       execInfo.pNodeList = extractNodeListFromStream(pMnode);
       execInfo.ts = ts;
-      mDebug("create trans successfully, update cached node list, numOfNodes:%d", taosArrayGetSize(execInfo.pNodeList));
+      mDebug("create trans successfully, update cached node list, numOfNodes:%d",
+             (int)taosArrayGetSize(execInfo.pNodeList));
     } else {
       mError("unexpected code during create nodeUpdate trans, code:%s", tstrerror(code));
     }

From 545fa5ebcb0bedd8d9073213cd145679cf3cc212 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 5 Jun 2024 11:38:38 +0800
Subject: [PATCH 131/132] add ts column for last cache

---
 source/libs/planner/src/planOptimizer.c |   6 +-
 tests/script/tsim/query/cache_last.sim  | 243 ++++++++++++++++++++++++
 2 files changed, 247 insertions(+), 2 deletions(-)

diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 7d10c02529..e9861c29b0 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -4159,8 +4159,10 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
 
     lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, pLastRowCols, false, cxt.pkBytes);
     lastRowScanOptRemoveUslessTargets(pScan->node.pTargets, cxt.pLastCols, cxt.pOtherCols, pLastRowCols);
-    if (pPKTsCol && ((pScan->node.pTargets->length == 1) || (pScan->node.pTargets->length == 2 && cxt.pkBytes > 0))) {
-      // when select last(ts),ts from ..., we add another ts to targets
+    if (pPKTsCol &&
+        ((cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pPKTsCol, nodesListGetNode(cxt.pLastCols, 0))) ||
+         (pScan->node.pTargets->length == 2 && cxt.pkBytes > 0))) {
+      // when select last(ts),tbname,ts from ..., we add another ts to targets
       sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol);
       nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pPKTsCol));
     }
diff --git a/tests/script/tsim/query/cache_last.sim b/tests/script/tsim/query/cache_last.sim
index 50199117b0..b2d5ad8aa0 100644
--- a/tests/script/tsim/query/cache_last.sim
+++ b/tests/script/tsim/query/cache_last.sim
@@ -101,5 +101,248 @@ if $rows != 1 then
   return -1
 endi
 
+print step 2-------------------------------
+
+sql drop database if exists test;
+sql create database test  cachemodel 'both';
+sql use test;
+sql create table stb (ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
+
+sql create table t1 using stb tags(1,1,1);
+sql create table t2 using stb tags(2,2,2);
+sql insert into t1 values('2024-06-05 11:00:00',1,2,3);
+sql insert into t1 values('2024-06-05 12:00:00',2,2,3);
+sql insert into t2 values('2024-06-05 13:00:00',3,2,3);
+sql insert into t2 values('2024-06-05 14:00:00',4,2,3);
+
+sql select last(ts) ts1,ts from stb;
+
+if $data00 != $data01 then
+  print $data00
+  return -1
+endi
+
+sql select last(ts) ts1,ts from stb group by tbname;
+
+if $data00 != $data01 then
+  print $data00
+  return -1
+endi
+
+sql select last(ts) ts1,tbname, ts from stb;
+
+if $data00 != $data02 then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+sql select last(ts) ts1,tbname, ts from stb group by tbname;
+
+if $data00 != $data02 then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+print step 3-------------------------------
+
+sql drop database if exists test1;
+sql create database test1  cachemodel 'both';
+sql use test1;
+sql create table stb (ts timestamp,a int primary key,b int,c int) tags(ta int,tb int,tc int);
+
+sql create table t1 using stb tags(1,1,1);
+sql create table t2 using stb tags(2,2,2);
+sql insert into t1 values('2024-06-05 11:00:00',1,2,3);
+sql insert into t1 values('2024-06-05 12:00:00',2,2,3);
+sql insert into t2 values('2024-06-05 13:00:00',3,2,3);
+sql insert into t2 values('2024-06-05 14:00:00',4,2,3);
+
+sql select last(ts) ts1,ts from stb;
+
+if $data00 != $data01 then
+  print $data00
+  return -1
+endi
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(ts) ts1,ts from stb group by tbname;
+
+if $data00 != $data01 then
+  print $data00
+  return -1
+endi
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(ts) ts1,tbname, ts from stb;
+
+if $data00 != $data02 then
+  print $data00
+  return -1
+endi
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+sql select last(ts) ts1,tbname, ts from stb group by tbname;
+
+if $data00 != $data02 then
+  print $data00
+  return -1
+endi
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+print step 4-------------------------------
+
+sql select last(a) a,ts from stb;
+
+if $data00 != 4 then
+  print $data00
+  return -1
+endi
+
+if $data01 != @24-06-05 14:00:00.000@ then
+  print $data01
+  return -1
+endi
+
+sql select last(a) a,ts from stb group by tbname;
+
+if $data00 != 4 then
+  print $data00
+  return -1
+endi
+
+if $data01 != @24-06-05 14:00:00.000@ then
+  print $data01
+  return -1
+endi
+
+sql select last(a) a,tbname, ts from stb;
+
+if $data00 != 4 then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+if $data02 != @24-06-05 14:00:00.000@ then
+  print $data02
+  return -1
+endi
+
+sql select last(a) a,tbname, ts from stb group by tbname;
+
+if $data00 != 4 then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+if $data02 != @24-06-05 14:00:00.000@ then
+  print $data02
+  return -1
+endi
+
+print step 5-------------------------------
+
+sql select last(ts) ts1,a from stb;
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+if $data01 != 4 then
+  print $data01
+  return -1
+endi
+
+sql select last(ts) ts1,a from stb group by tbname;
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+if $data01 != 4 then
+  print $data01
+  return -1
+endi
+
+sql select last(ts) ts1,tbname, a from stb;
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+if $data02 != 4 then
+  print $data02
+  return -1
+endi
+
+sql select last(ts) ts1,tbname, a from stb group by tbname;
+
+if $data00 != @24-06-05 14:00:00.000@ then
+  print $data00
+  return -1
+endi
+
+if $data01 != t2 then
+  print $data01
+  return -1
+endi
+
+if $data02 != 4 then
+  print $data02
+  return -1
+endi
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 1e33a1f4088aefbdb0432a82ff5e2aedee6adefa Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 5 Jun 2024 11:43:39 +0800
Subject: [PATCH 132/132] fix(stream): fix memory leak.

---
 source/dnode/mnode/impl/src/mndStream.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 9bfda65108..9c8f3f26ff 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -139,6 +139,7 @@ int32_t mndInitStream(SMnode *pMnode) {
 
 void mndCleanupStream(SMnode *pMnode) {
   taosArrayDestroy(execInfo.pTaskList);
+  taosArrayDestroy(execInfo.pNodeList);
   taosHashCleanup(execInfo.pTaskMap);
   taosHashCleanup(execInfo.transMgmt.pDBTrans);
   taosHashCleanup(execInfo.pTransferStateStreams);