From f5b52749c288eff98799dd0f8db5dd677a84f688 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 9 Apr 2023 01:39:09 +0800
Subject: [PATCH 01/65] refactor: do some internal refactor.

---
 include/libs/executor/executor.h              |  16 +-
 include/libs/qcom/query.h                     |   1 +
 include/libs/stream/tstream.h                 |  44 +--
 include/libs/wal/wal.h                        |   1 +
 source/client/inc/clientInt.h                 |  36 +--
 source/client/src/clientImpl.c                |   4 +-
 source/client/src/clientTmq.c                 |  47 ++-
 source/dnode/mnode/impl/src/mndDef.c          |  10 +-
 source/dnode/mnode/impl/src/mndScheduler.c    |   8 +-
 source/dnode/mnode/impl/src/mndStream.c       |   4 +-
 source/dnode/snode/src/snode.c                |  11 +-
 source/dnode/vnode/inc/vnode.h                |   6 +-
 source/dnode/vnode/src/inc/tq.h               |   2 +-
 source/dnode/vnode/src/meta/metaCache.c       |  11 +-
 source/dnode/vnode/src/sma/smaRollup.c        |   2 +-
 source/dnode/vnode/src/tq/tq.c                | 284 +++++++++++++-----
 source/dnode/vnode/src/tq/tqMeta.c            |  10 +-
 source/dnode/vnode/src/tq/tqOffset.c          |  25 +-
 source/dnode/vnode/src/tq/tqPush.c            |   3 +-
 source/dnode/vnode/src/tq/tqRead.c            |  85 ++++--
 source/dnode/vnode/src/tq/tqScan.c            |  18 +-
 source/dnode/vnode/src/tq/tqSink.c            |  12 +-
 source/dnode/vnode/src/vnd/vnodeSvr.c         |   2 -
 source/libs/executor/src/executor.c           |  54 ++--
 source/libs/executor/src/executorimpl.c       |   6 +-
 source/libs/executor/src/scanoperator.c       |  11 +-
 source/libs/executor/src/timewindowoperator.c |   7 +-
 source/libs/stream/inc/streamInc.h            |   2 +-
 source/libs/stream/src/stream.c               |  43 ++-
 source/libs/stream/src/streamData.c           |  25 +-
 source/libs/stream/src/streamDispatch.c       |  26 +-
 source/libs/stream/src/streamExec.c           | 120 ++++----
 source/libs/stream/src/streamMeta.c           |  19 +-
 source/libs/stream/src/streamRecover.c        |  36 +--
 source/libs/stream/src/streamState.c          |   2 +-
 source/libs/stream/src/streamTask.c           |  51 +++-
 36 files changed, 625 insertions(+), 419 deletions(-)

diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 33172a4f86..ee8ee1050d 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -26,6 +26,7 @@ extern "C" {
 
 typedef void* qTaskInfo_t;
 typedef void* DataSinkHandle;
+
 struct SRpcMsg;
 struct SSubplan;
 
@@ -118,7 +119,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
  * @param isAdd
  * @return
  */
-int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd);
+int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd, SArray* pList);
 
 /**
  * Create the exec task object according to task json
@@ -162,6 +163,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
  * @return
  */
 int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode);
+
 int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode);
 
 bool qTaskIsExecuting(qTaskInfo_t qinfo);
@@ -181,21 +183,11 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len);
 int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len);
 
 STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key);
-/**
- * return the scan info, in the form of tuple of two items, including table uid and current timestamp
- * @param tinfo
- * @param uid
- * @param ts
- * @return
- */
-int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
 
-int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
+SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo);
 
 int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType);
 
-// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver);
-//
 int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit);
 
 void qStreamSetOpen(qTaskInfo_t tinfo);
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index b6ada5a0c7..cfc6ef2025 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -194,6 +194,7 @@ typedef struct SRequestConnInfo {
 
 typedef void (*__freeFunc)(void* param);
 
+// todo add creator/destroyer function
 typedef struct SMsgSendInfo {
   __async_send_cb_fn_t fp;      // async callback function
   STargetInfo          target;  // for update epset
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 5b1d1fa1bc..d9b82c8c59 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -225,15 +225,15 @@ static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) {
 
 void* streamQueueNextItem(SStreamQueue* queue);
 
-SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit);
+SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type);
 void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit);
 
 SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit);
 
 typedef struct {
-  char* qmsg;
-  // followings are not applicable to encoder and decoder
-  void* executor;
+  char*             qmsg;
+  void*             pExecutor;   // not applicable to encoder and decoder
+  struct STqReader* pTqReader;  // not applicable to encoder and decoder
 } STaskExec;
 
 typedef struct {
@@ -280,16 +280,20 @@ typedef struct {
   SEpSet  epSet;
 } SStreamChildEpInfo;
 
-struct SStreamTask {
-  int64_t streamId;
-  int32_t taskId;
-  int32_t totalLevel;
-  int8_t  taskLevel;
-  int8_t  outputType;
-  int16_t dispatchMsgType;
+typedef struct SStreamId {
+  int64_t     streamId;
+  int32_t     taskId;
+  const char* idStr;
+} SStreamId;
 
-  int8_t taskStatus;
-  int8_t schedStatus;
+struct SStreamTask {
+  SStreamId id;
+  int32_t   totalLevel;
+  int8_t    taskLevel;
+  int8_t    outputType;
+  int16_t   dispatchMsgType;
+  int8_t    taskStatus;
+  int8_t    schedStatus;
 
   // node info
   int32_t selfChildId;
@@ -319,11 +323,8 @@ struct SStreamTask {
     STaskSinkFetch         fetchSink;
   };
 
-  int8_t inputStatus;
-  int8_t outputStatus;
-
-  // STaosQueue*   inputQueue1;
-  // STaosQall*    inputQall;
+  int8_t        inputStatus;
+  int8_t        outputStatus;
   SStreamQueue* inputQueue;
   SStreamQueue* outputQueue;
 
@@ -345,8 +346,8 @@ struct SStreamTask {
   SArray* checkReqIds;  // shuffle
   int32_t refCnt;
 
-  int64_t checkpointingId;
-  int32_t checkpointAlignCnt;
+  int64_t    checkpointingId;
+  int32_t    checkpointAlignCnt;
 };
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -355,8 +356,9 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
 SStreamTask* tNewSStreamTask(int64_t streamId);
 int32_t      tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
 int32_t      tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
-void         tFreeSStreamTask(SStreamTask* pTask);
+void         tFreeStreamTask(SStreamTask* pTask);
 int32_t      tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem);
+bool         tInputQueueIsFull(const SStreamTask* pTask);
 
 static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) {
   atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index ccbc53fa5d..fdd21c7092 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -138,6 +138,7 @@ typedef struct {
   int8_t enableRef;
 } SWalFilterCond;
 
+// todo hide this struct
 typedef struct {
   SWal          *pWal;
   int64_t        readerId;
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 86db35b412..41f87379a9 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -36,14 +36,6 @@ extern "C" {
 
 #include "tconfig.h"
 
-#define CHECK_CODE_GOTO(expr, label) \
-  do {                               \
-    code = expr;                     \
-    if (TSDB_CODE_SUCCESS != code) { \
-      goto label;                    \
-    }                                \
-  } while (0)
-
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
 #define HEARTBEAT_INTERVAL         1500  // ms
 
@@ -286,28 +278,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   return (SReqResultInfo*)&msg->resInfo;
 }
 
-static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
-  SMqRspObj* pRspObj = (SMqRspObj*)res;
-  pRspObj->resIter++;
-
-  if (pRspObj->resIter < pRspObj->rsp.blockNum) {
-    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
-    if (pRspObj->rsp.withSchema) {
-      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
-      setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
-      taosMemoryFreeClear(pRspObj->resInfo.row);
-      taosMemoryFreeClear(pRspObj->resInfo.pCol);
-      taosMemoryFreeClear(pRspObj->resInfo.length);
-      taosMemoryFreeClear(pRspObj->resInfo.convertBuf);
-      taosMemoryFreeClear(pRspObj->resInfo.convertJson);
-    }
-
-    setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false);
-    return &pRspObj->resInfo;
-  }
-
-  return NULL;
-}
+SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4);
 
 static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) {
   if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo);
@@ -320,7 +291,6 @@ extern int32_t  clientConnRefPool;
 extern int32_t  timestampDeltaLimit;
 extern int64_t  lastClusterId;
 
-
 __async_send_cb_fn_t getMsgRspHandle(int32_t msgType);
 
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
@@ -373,7 +343,6 @@ void taos_close_internal(void* taos);
 // global, called by mgmt
 int  hbMgrInit();
 void hbMgrCleanUp();
-int  hbHandleRsp(SClientHbBatchRsp* hbRsp);
 
 // cluster level
 SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
@@ -386,9 +355,6 @@ void       stopAllRequests(SHashObj* pRequests);
 int  hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
 void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
 
-// --- mq
-void hbMgrInitMqHbRspHandle();
-
 typedef struct SSqlCallbackWrapper {
   SParseContext* pParseCtx;
   SCatalogReq*   pCatalogReq;
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 9e5d9080b4..8ced67c9b2 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1039,8 +1039,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
                       .sysInfo = pRequest->pTscObj->sysInfo,
                       .allocatorId = pRequest->allocatorRefId};
 
-  SAppInstInfo* pAppInfo = getAppInfo(pRequest);
-  SQueryPlan*   pDag = NULL;
+  SQueryPlan* pDag = NULL;
 
   int64_t st = taosGetTimestampUs();
   int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
@@ -1052,7 +1051,6 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
   }
 
   pRequest->metric.execStart = taosGetTimestampUs();
-
   pRequest->metric.planCostUs = pRequest->metric.execStart - st;
 
   if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index e295ec93af..f90c004eec 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -154,7 +154,7 @@ typedef struct {
 
 typedef struct {
   int8_t          tmqRspType;
-  int32_t         epoch;         // epoch can be used to guard the vgHandle
+  int32_t         epoch;  // epoch can be used to guard the vgHandle
   int32_t         vgId;
   SMqClientVg*    vgHandle;
   SMqClientTopic* topicHandle;
@@ -210,6 +210,11 @@ typedef struct {
   tmq_t*               pTmq;
 } SMqCommitCbParam;
 
+typedef struct SSyncCommitInfo {
+  tsem_t  sem;
+  int32_t code;
+} SSyncCommitInfo;
+
 static int32_t doAskEp(tmq_t* tmq);
 static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg);
 static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet);
@@ -521,11 +526,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  pMsgSendInfo->msgInfo = (SDataBuf){
-      .pData = buf,
-      .len = sizeof(SMsgHead) + len,
-      .handle = NULL,
-  };
+  pMsgSendInfo->msgInfo = (SDataBuf) { .pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL };
 
   pMsgSendInfo->requestId = generateRequestId();
   pMsgSendInfo->requestObjRefId = 0;
@@ -786,11 +787,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
     goto OVER;
   }
 
-  sendInfo->msgInfo = (SDataBuf){
-      .pData = pReq,
-      .len = tlen,
-      .handle = NULL,
-  };
+  sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = tlen, .handle = NULL };
 
   sendInfo->requestId = generateRequestId();
   sendInfo->requestObjRefId = 0;
@@ -2115,11 +2112,6 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void*
   }
 }
 
-typedef struct SSyncCommitInfo {
-  tsem_t  sem;
-  int32_t code;
-} SSyncCommitInfo;
-
 static void commitCallBackFn(tmq_t *pTmq, int32_t code, void* param) {
   SSyncCommitInfo* pInfo = (SSyncCommitInfo*) param;
   pInfo->code = code;
@@ -2298,3 +2290,26 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons
              waitingRspNum);
   }
 }
+
+SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
+  SMqRspObj* pRspObj = (SMqRspObj*)res;
+  pRspObj->resIter++;
+
+  if (pRspObj->resIter < pRspObj->rsp.blockNum) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
+    if (pRspObj->rsp.withSchema) {
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
+      setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
+      taosMemoryFreeClear(pRspObj->resInfo.row);
+      taosMemoryFreeClear(pRspObj->resInfo.pCol);
+      taosMemoryFreeClear(pRspObj->resInfo.length);
+      taosMemoryFreeClear(pRspObj->resInfo.convertBuf);
+      taosMemoryFreeClear(pRspObj->resInfo.convertJson);
+    }
+
+    setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false);
+    return &pRspObj->resInfo;
+  }
+
+  return NULL;
+}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index fb81a764f1..e221a64619 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -158,7 +158,10 @@ void tFreeStreamObj(SStreamObj *pStream) {
   taosMemoryFree(pStream->sql);
   taosMemoryFree(pStream->ast);
   taosMemoryFree(pStream->physicalPlan);
-  if (pStream->outputSchema.nCols) taosMemoryFree(pStream->outputSchema.pSchema);
+
+  if (pStream->outputSchema.nCols) {
+    taosMemoryFree(pStream->outputSchema.pSchema);
+  }
 
   int32_t sz = taosArrayGetSize(pStream->tasks);
   for (int32_t i = 0; i < sz; i++) {
@@ -166,11 +169,14 @@ void tFreeStreamObj(SStreamObj *pStream) {
     int32_t taskSz = taosArrayGetSize(pLevel);
     for (int32_t j = 0; j < taskSz; j++) {
       SStreamTask *pTask = taosArrayGetP(pLevel, j);
-      tFreeSStreamTask(pTask);
+      tFreeStreamTask(pTask);
     }
+
     taosArrayDestroy(pLevel);
   }
+
   taosArrayDestroy(pStream->tasks);
+
   // tagSchema.pSchema
   if (pStream->tagSchema.nCols > 0) {
     taosMemoryFree(pStream->tagSchema.pSchema);
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index d1671aa12a..504749df49 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -138,7 +138,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream
       for (int32_t j = 0; j < sinkLvSize; j++) {
         SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
         if (pLastLevelTask->nodeId == pVgInfo->vgId) {
-          pVgInfo->taskId = pLastLevelTask->taskId;
+          pVgInfo->taskId = pLastLevelTask->id.taskId;
           break;
         }
       }
@@ -149,7 +149,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream
     SArray* pArray = taosArrayGetP(pStream->tasks, 0);
     // one sink only
     SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
-    pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
+    pTask->fixedEpDispatcher.taskId = lastLevelTask->id.taskId;
     pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
     pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
   }
@@ -440,7 +440,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
       pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
       pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH;
 
-      pTask->fixedEpDispatcher.taskId = pInnerTask->taskId;
+      pTask->fixedEpDispatcher.taskId = pInnerTask->id.taskId;
       pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId;
       pTask->fixedEpDispatcher.epSet = pInnerTask->epSet;
 
@@ -460,7 +460,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
       pEpInfo->childId = pTask->selfChildId;
       pEpInfo->epSet = pTask->epSet;
       pEpInfo->nodeId = pTask->nodeId;
-      pEpInfo->taskId = pTask->taskId;
+      pEpInfo->taskId = pTask->id.taskId;
       taosArrayPush(pInnerTask->childEpInfo, &pEpInfo);
       sdbRelease(pSdb, pVgroup);
     }
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index b7f80f6b0e..ab83f29ef9 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -600,7 +600,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
     return -1;
   }
   pReq->head.vgId = htonl(pTask->nodeId);
-  pReq->taskId = pTask->taskId;
+  pReq->taskId = pTask->id.taskId;
   STransAction action = {0};
   memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet));
   action.pCont = pReq;
@@ -1208,7 +1208,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
 
         // task id
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->taskId, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->id.taskId, false);
 
         // node type
         char nodeType[20 + VARSTR_HEADER_SIZE] = {0};
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 3d1b356f8c..447c90eb58 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -83,14 +83,11 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
     return -1;
   }
 
-  SReadHandle mgHandle = {
-      .vnode = NULL,
-      .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo),
-      .pStateBackend = pTask->pState,
-  };
+  int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo);
+  SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState };
 
-  pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0);
-  ASSERT(pTask->exec.executor);
+  pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0);
+  ASSERT(pTask->exec.pExecutor);
 
   streamSetupTrigger(pTask);
   return 0;
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index a9e5fe628b..d62eebd2e1 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -256,15 +256,15 @@ void       tqCloseReader(STqReader *);
 
 void    tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
 int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
-int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
+int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
 int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
 
 int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
 void tqNextBlock(STqReader *pReader, SFetchRet *ret);
 
-int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
+int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
 // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver);
-bool    tqNextDataBlock2(STqReader *pReader);
+bool    tqNextDataBlock(STqReader *pReader);
 bool    tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids);
 int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet);
 int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index 9037644602..884c01d397 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -80,7 +80,7 @@ typedef struct {
 
 typedef struct {
   int8_t       subType;
-  STqReader*   pExecReader;
+  STqReader*   pTqReader;
   qTaskInfo_t  task;
   union {
     STqExecCol execCol;
diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c
index 9501bf4b8e..795f281ab2 100644
--- a/source/dnode/vnode/src/meta/metaCache.c
+++ b/source/dnode/vnode/src/meta/metaCache.c
@@ -531,10 +531,11 @@ static void freePayload(const void* key, size_t keyLen, void* value) {
     return;
   }
 
-  SHashObj*            pHashObj = (SHashObj*)p[0];
+  SHashObj* pHashObj = (SHashObj*)p[0];
+
   STagFilterResEntry** pEntry = taosHashGet(pHashObj, &p[1], sizeof(uint64_t));
 
-  {
+  if (pEntry != NULL && (*pEntry) != NULL) {
     int64_t st = taosGetTimestampUs();
 
     SListIter iter = {0};
@@ -547,9 +548,9 @@ static void freePayload(const void* key, size_t keyLen, void* value) {
         void* tmp = tdListPopNode(&((*pEntry)->list), pNode);
         taosMemoryFree(tmp);
 
-        int64_t et = taosGetTimestampUs();
-        metaInfo("clear items in cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)),
-                 (et - st) / 1000.0);
+        double el = (taosGetTimestampUs() - st) / 1000.0;
+        metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)),
+                 el);
         break;
       }
     }
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index c75c675ec3..8aeb705d90 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -168,7 +168,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids,
 
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pRSmaInfo->taskInfo[i]) {
-      if ((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) {
+      if ((terrno = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd, NULL)) < 0) {
         tdReleaseRSmaInfo(pSma, pRSmaInfo);
         smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i,
                  terrstr());
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 66e6ac4cd8..44d5e26603 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -15,6 +15,8 @@
 
 #include "tq.h"
 
+#define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)
+
 int32_t tqInit() {
   int8_t old;
   while (1) {
@@ -57,12 +59,12 @@ static void destroyTqHandle(void* data) {
   if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
     taosMemoryFreeClear(pData->execHandle.execCol.qmsg);
   } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) {
-    tqCloseReader(pData->execHandle.pExecReader);
+    tqCloseReader(pData->execHandle.pTqReader);
     walCloseReader(pData->pWalReader);
     taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid);
   } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
     walCloseReader(pData->pWalReader);
-    tqCloseReader(pData->execHandle.pExecReader);
+    tqCloseReader(pData->execHandle.pTqReader);
   }
 }
 
@@ -78,12 +80,33 @@ static void tqPushEntryFree(void* data) {
   taosMemoryFree(p);
 }
 
+static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) {
+  return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG &&
+         pLeft->val.version <= pRight->val.version;
+}
+
+// stream_task:stream_id:task_id
+static void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
+  int32_t n = 12;
+  char* p = dst;
+
+  memcpy(p, "stream_task:", n);
+  p += n;
+
+  int32_t inc = tintToHex(streamId, p);
+  p += inc;
+
+  *(p++) = ':';
+  tintToHex(taskId, p);
+}
+
 STQ* tqOpen(const char* path, SVnode* pVnode) {
   STQ* pTq = taosMemoryCalloc(1, sizeof(STQ));
   if (pTq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
+
   pTq->path = taosStrdup(path);
   pTq->pVnode = pVnode;
   pTq->walLogLastVer = pVnode->pWal->vers.lastVer;
@@ -249,11 +272,6 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
   return 0;
 }
 
-static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) {
-  return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG &&
-         pLeft->val.version <= pRight->val.version;
-}
-
 int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
   STqOffset offset = {0};
   int32_t vgId = TD_VID(pTq->pVnode);
@@ -432,8 +450,6 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
   return 0;
 }
 
-#define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)
-
 static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
                                                    SRpcMsg* pMsg, STqOffsetVal* pOffset) {
   uint64_t consumerId = pRequest->consumerId;
@@ -805,10 +821,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
           qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, req.newConsumerId);
       void* scanner = NULL;
       qExtractStreamScanner(pHandle->execHandle.task, &scanner);
-      pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
+      pHandle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner);
     } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
       pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL);
-      pHandle->execHandle.pExecReader = tqOpenReader(pVnode);
+      pHandle->execHandle.pTqReader = tqOpenReader(pVnode);
 
       pHandle->execHandle.execDb.pFilterOutTbUid =
           taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
@@ -827,8 +843,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
         int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
         tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid);
       }
-      pHandle->execHandle.pExecReader = tqOpenReader(pVnode);
-      tqReaderSetTbUidList(pHandle->execHandle.pExecReader, tbUidList);
+      pHandle->execHandle.pTqReader = tqOpenReader(pVnode);
+      tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList);
       taosArrayDestroy(tbUidList);
 
       buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
@@ -886,16 +902,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
 }
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
-#if 0
-  if (pTask->taskLevel == TASK_LEVEL__AGG) {
-    A(taosArrayGetSize(pTask->childEpInfo) != 0);
-  }
-#endif
+  // todo extract method
+  char buf[128] = {0};
+  sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId);
 
   int32_t vgId = TD_VID(pTq->pVnode);
+  pTask->id.idStr = taosStrdup(buf);
   pTask->refCnt = 1;
   pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
-
   pTask->inputQueue = streamQueueOpen();
   pTask->outputQueue = streamQueueOpen();
 
@@ -920,14 +934,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
     }
 
     SReadHandle handle = {
-        .meta = pTq->pVnode->pMeta,
-        .vnode = pTq->pVnode,
-        .initTqReader = 1,
-        .pStateBackend = pTask->pState,
-    };
+        .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState};
 
-    pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId);
-    if (pTask->exec.executor == NULL) {
+    pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId);
+    if (pTask->exec.pExecutor == NULL) {
       return -1;
     }
 
@@ -936,14 +946,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
     if (pTask->pState == NULL) {
       return -1;
     }
-    SReadHandle mgHandle = {
-        .vnode = NULL,
-        .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo),
-        .pStateBackend = pTask->pState,
-    };
 
-    pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId);
-    if (pTask->exec.executor == NULL) {
+    int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo);
+    SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState};
+
+    pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId);
+    if (pTask->exec.pExecutor == NULL) {
       return -1;
     }
   }
@@ -964,15 +972,26 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
       ver1 = info.skmVer;
     }
 
-    pTask->tbSink.pTSchema =
-        tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, ver1);
+    SSchemaWrapper* pschemaWrapper = pTask->tbSink.pSchemaWrapper;
+    pTask->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
     if(pTask->tbSink.pTSchema == NULL) {
       return -1;
     }
   }
 
+  if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
+    pTask->exec.pTqReader = tqOpenReader(pTq->pVnode);
+    if (pTask->exec.pTqReader == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    SArray* pList = qGetQueriedTableListInfo(pTask->exec.pExecutor);
+    tqReaderAddTbUidList(pTask->exec.pTqReader, pList);
+  }
+
   streamSetupTrigger(pTask);
-  tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId, pTask->taskLevel);
+  tqInfo("vgId:%d expand stream task, s-task:%s, child id %d, level %d", vgId, pTask->id.idStr, pTask->selfChildId, pTask->taskLevel);
   return 0;
 }
 
@@ -995,6 +1014,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
       .upstreamNodeId = req.upstreamNodeId,
       .upstreamTaskId = req.upstreamTaskId,
   };
+
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
   if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) {
     rsp.status = 1;
@@ -1085,6 +1105,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
     taosMemoryFree(pTask);
     return -1;
   }
+
   tDecoderClear(&decoder);
 
   // 2.save task
@@ -1298,7 +1319,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
     SStreamTask* pTask = *(SStreamTask**)pIter;
     if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue;
 
-    qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver);
+    qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver);
 
     if (!failed) {
       SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
@@ -1308,7 +1329,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
       atomic_add_fetch_32(pRefBlock->dataRef, 1);
 
       if (tAppendDataForStream(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
-        qError("stream task input del failed, task id %d", pTask->taskId);
+        qError("stream task input del failed, task id %d", pTask->id.taskId);
 
         atomic_sub_fetch_32(pRef, 1);
         taosFreeQitem(pRefBlock);
@@ -1316,7 +1337,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
       }
 
       if (streamSchedExec(pTask) < 0) {
-        qError("stream task launch failed, task id %d", pTask->taskId);
+        qError("stream task launch failed, task id %d", pTask->id.taskId);
         continue;
       }
 
@@ -1343,12 +1364,12 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
 
     if (!failed) {
       if (tAppendDataForStream(pTask, (SStreamQueueItem*)pStreamBlock) < 0) {
-        qError("stream task input del failed, task id %d", pTask->taskId);
+        qError("stream task input del failed, task id %d", pTask->id.taskId);
         continue;
       }
 
       if (streamSchedExec(pTask) < 0) {
-        qError("stream task launch failed, task id %d", pTask->taskId);
+        qError("stream task launch failed, task id %d", pTask->id.taskId);
         continue;
       }
     } else {
@@ -1361,15 +1382,82 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   return 0;
 }
 
+static int32_t doAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
+  int32_t code = tAppendDataForStream(pTask, pQueueItem);
+  if (code < 0) {
+    tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver);
+    return -1;
+  }
+
+  if (streamSchedExec(pTask) < 0) {
+    tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno));
+    return -1;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
+  STqOffset offset = {0};
+  tqOffsetResetToLog(&offset.val, ver);
+
+  tstrncpy(offset.subKey, pKey, tListLen(offset.subKey));
+
+  // keep the offset info in the offset store
+  tqOffsetWrite(pOffsetStore, &offset);
+}
+
+static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit,
+                                         const char* key, int64_t ver) {
+  doSaveTaskOffset(pOffsetStore, key, ver);
+  int32_t code = doAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver);
+
+  // remove the offset, if all functions are completed successfully.
+  if (code == TSDB_CODE_SUCCESS) {
+    tqOffsetDelete(pOffsetStore, key);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static void saveOffsetForAllTasks(STQ* pTq, SPackedData submit) {
+  void* pIter = NULL;
+
+  while(1) {
+    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
+    if (pIter == NULL) {
+      break;
+    }
+
+    SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      continue;
+    }
+
+    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
+              pTask->taskStatus);
+      continue;
+    }
+
+    char key[128] = {0};
+    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
+
+    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
+    if (pOffset == NULL) {
+      doSaveTaskOffset(pTq->pOffsetStore, key, submit.ver);
+    }
+  }
+}
+
 int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
   void* pIter = NULL;
-  bool  succ = true;
 
-  SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit);
+  SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit, STREAM_INPUT__DATA_SUBMIT);
   if (pSubmit == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tqError("failed to create data submit for stream since out of memory");
-    succ = false;
+    saveOffsetForAllTasks(pTq, submit);
+    return -1;
   }
 
   while (1) {
@@ -1384,35 +1472,85 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
     }
 
     if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->taskId, pTask->taskStatus);
+      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
+              pTask->taskStatus);
       continue;
     }
 
-    tqDebug("data submit enqueue stream task:%d, ver: %" PRId64, pTask->taskId, submit.ver);
-    if (succ) {
-      int32_t code = tAppendDataForStream(pTask, (SStreamQueueItem*)pSubmit);
-      if (code < 0) {
-        // let's handle the back pressure
+    // check if offset value exists
+    char key[128] = {0};
+    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
 
-        tqError("stream task:%d failed to put into queue for, too many", pTask->taskId);
-        continue;
+    if (tInputQueueIsFull(pTask)) {
+      STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
+
+      int64_t ver = submit.ver;
+      if (pOffset == NULL) {
+        doSaveTaskOffset(pTq->pOffsetStore, key, submit.ver);
+      } else {
+        ver = pOffset->val.version;
       }
 
-      if (streamSchedExec(pTask) < 0) {
-        tqError("stream task:%d launch failed, code:%s", pTask->taskId, tstrerror(terrno));
-        continue;
+      tqDebug("s-task:%s input queue is full, do nothing, start ver:%" PRId64, pTask->id.idStr, ver);
+      continue;
+    }
+
+    // check if offset value exists
+    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
+    if (pOffset != NULL) {
+      // seek the stored version and extract data from WAL
+      int32_t code = tqSeekVer(pTask->exec.pTqReader, pOffset->val.version, "");
+
+      // all data has been retrieved from WAL, let's try submit block directly.
+      if (code == TSDB_CODE_SUCCESS) {  // all data retrieved, abort
+        // append the data for the stream
+        SFetchRet ret = {0};
+        terrno = 0;
+
+        tqNextBlock(pTask->exec.pTqReader, &ret);
+        if (ret.fetchType == FETCH_TYPE__DATA) {
+          SStreamDataBlock* pBlocks = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
+          if (pBlocks == NULL) { // failed, do nothing
+            terrno = TSDB_CODE_OUT_OF_MEMORY;
+            continue;
+          }
+
+          ret.data.info.type = STREAM_NORMAL;
+          pBlocks->type = STREAM_INPUT__DATA_BLOCK;
+          pBlocks->sourceVer = pOffset->val.version;
+          pBlocks->blocks = taosArrayInit(0, sizeof(SSDataBlock));
+          taosArrayPush(pBlocks->blocks, &ret.data);
+
+          int64_t* ts = (int64_t*)(((SColumnInfoData*)ret.data.pDataBlock->pData)->pData);
+//          tqDebug("-----------%ld\n", ts[0]);
+
+          code = doAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pBlocks, pBlocks->sourceVer);
+          if (code == TSDB_CODE_SUCCESS) {
+            pOffset->val.version = pTask->exec.pTqReader->pWalReader->curVersion;
+            tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
+                    pOffset->val.version);
+          }
+
+        } else {  // FETCH_TYPE__NONE, let's try submit block directly
+          tqDebug("s-task:%s data in WAL are all consumed, try data in submit message", pTask->id.idStr);
+          addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
+        }
+
+        // do nothing if failed, since the offset value is kept already
+      } else {  // failed to seek to the WAL version
+        // todo handle the case where offset has been deleted in WAL, due to stream computing too slow
+        tqDebug("s-task:%s data in WAL are all consumed, try data in submit msg", pTask->id.idStr);
+        addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
       }
     } else {
-      streamTaskInputFail(pTask);
+      addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
     }
   }
 
-  if (pSubmit != NULL) {
-    streamDataSubmitDestroy(pSubmit);
-    taosFreeQitem(pSubmit);
-  }
+  streamDataSubmitDestroy(pSubmit);
+  taosFreeQitem(pSubmit);
 
-  return succ ? 0 : -1;
+  return 0;
 }
 
 int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
@@ -1420,6 +1558,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   int32_t            taskId = pReq->taskId;
   SStreamTask*       pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
   if (pTask) {
+    tqDebug("stream task:%d start to process run req", pTask->id.taskId);
     streamProcessRunReq(pTask);
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     return 0;
@@ -1456,7 +1595,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
   SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   int32_t             taskId = ntohl(pRsp->upstreamTaskId);
   SStreamTask*        pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
-  tqDebug("recv dispatch rsp, code: %x", pMsg->code);
+  tqDebug("recv dispatch rsp, code:%x", pMsg->code);
   if (pTask) {
     streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@@ -1484,10 +1623,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) {
   int32_t      taskId = req.dstTaskId;
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
   if (pTask) {
-    SRpcMsg rsp = {
-        .info = pMsg->info,
-        .code = 0,
-    };
+    SRpcMsg rsp = { .info = pMsg->info, .code = 0 };
     streamProcessRetrieveReq(pTask, &req, &rsp);
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     tDeleteStreamRetrieveReq(&req);
@@ -1523,10 +1659,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
 
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
   if (pTask) {
-    SRpcMsg rsp = {
-        .info = pMsg->info,
-        .code = 0,
-    };
+    SRpcMsg rsp = { .info = pMsg->info, .code = 0 };
     streamProcessDispatchReq(pTask, &req, &rsp, false);
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     rpcFreeCont(pMsg->pCont);
@@ -1543,10 +1676,7 @@ FAIL:
 
   SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
   if (pRspHead == NULL) {
-    SRpcMsg rsp = {
-        .code = TSDB_CODE_OUT_OF_MEMORY,
-        .info = pMsg->info,
-    };
+    SRpcMsg rsp = { .code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info };
     tqDebug("send dispatch error rsp, code: %x", code);
     tmsgSendRsp(&rsp);
     rpcFreeCont(pMsg->pCont);
@@ -1564,11 +1694,7 @@ FAIL:
   pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL;
 
   SRpcMsg rsp = {
-      .code = code,
-      .info = pMsg->info,
-      .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp),
-      .pCont = pRspHead,
-  };
+      .code = code, .info = pMsg->info, .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), .pCont = pRspHead};
   tqDebug("send dispatch error rsp, code: %x", code);
   tmsgSendRsp(&rsp);
   rpcFreeCont(pMsg->pCont);
diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c
index a273f2edec..8f8ee78e97 100644
--- a/source/dnode/vnode/src/tq/tqMeta.c
+++ b/source/dnode/vnode/src/tq/tqMeta.c
@@ -320,15 +320,15 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
         code = -1;
         goto end;
       }
-      handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
-      if (handle.execHandle.pExecReader == NULL) {
+      handle.execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner);
+      if (handle.execHandle.pTqReader == NULL) {
         tqError("cannot extract exec reader for %s", handle.subKey);
         code = -1;
         goto end;
       }
     } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) {
       handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
-      handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode);
+      handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode);
 
       buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
                        (SSnapContext**)(&reader.sContext));
@@ -343,8 +343,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
         int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
         tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid);
       }
-      handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode);
-      tqReaderSetTbUidList(handle.execHandle.pExecReader, tbUidList);
+      handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode);
+      tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList);
       taosArrayDestroy(tbUidList);
 
       buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c
index 66d1ac2c7e..e8051a1406 100644
--- a/source/dnode/vnode/src/tq/tqOffset.c
+++ b/source/dnode/vnode/src/tq/tqOffset.c
@@ -128,31 +128,35 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) {
 }
 
 int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
-  if (!pStore->needCommit) return 0;
+  if (!pStore->needCommit) {
+    return 0;
+  }
+
   // TODO file name should be with a newer version
   char*     fname = tqOffsetBuildFName(pStore->pTq->path, 0);
   TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-
-    int32_t     err = terrno;
-    const char* errStr = tstrerror(err);
-    int32_t     sysErr = errno;
-    const char* sysErrStr = strerror(errno);
-    tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname,
-            sysErrStr);
+    const char* err = strerror(errno);
+    tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err);
     taosMemoryFree(fname);
     return -1;
   }
+
   taosMemoryFree(fname);
+
   void* pIter = NULL;
   while (1) {
     pIter = taosHashIterate(pStore->pHash, pIter);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
+
     STqOffset* pOffset = (STqOffset*)pIter;
     int32_t    bodyLen;
     int32_t    code;
     tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code);
+
     if (code < 0) {
       taosHashCancelIterate(pStore->pHash, pIter);
       return -1;
@@ -166,6 +170,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
     SEncoder encoder;
     tEncoderInit(&encoder, abuf, bodyLen);
     tEncodeSTqOffset(&encoder, pOffset);
+
     // write file
     int64_t writeLen;
     if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) {
@@ -174,8 +179,10 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
       taosMemoryFree(buf);
       return -1;
     }
+
     taosMemoryFree(buf);
   }
+
   // close and rename file
   taosCloseFile(&pFile);
   pStore->needCommit = 0;
diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c
index 9dfcf43e4d..dd003aec98 100644
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@@ -331,6 +331,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
     if (msgType == TDMT_VND_SUBMIT) {
       void* data = taosMemoryMalloc(len);
       if (data == NULL) {
+        // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", vgId);
         return -1;
@@ -339,7 +340,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
       memcpy(data, pReq, len);
       SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver};
 
-      tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq);
+      tqDebug("tq copy submit msg:%p len:%d ver:%" PRId64 " from %p for stream", data, len, ver, pReq);
       tqProcessSubmitReq(pTq, submit);
     }
 
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 54e4e393ec..24546168e5 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -113,7 +113,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
     }
 
     SMetaReader mr = {0};
-    metaReaderInit(&mr, pHandle->execHandle.pExecReader->pVnodeMeta, 0);
+    metaReaderInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0);
 
     if (metaGetTableEntryByName(&mr, req.tbName) < 0) {
       metaReaderClear(&mr);
@@ -262,8 +262,6 @@ STqReader* tqOpenReader(SVnode* pVnode) {
   }
 
   pReader->pVnodeMeta = pVnode->pMeta;
-  /*pReader->pMsg = NULL;*/
-//  pReader->ver = -1;
   pReader->pColIdList = NULL;
   pReader->cachedSchemaVer = 0;
   pReader->cachedSchemaSuid = 0;
@@ -296,10 +294,10 @@ void tqCloseReader(STqReader* pReader) {
 
 int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
   if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
-    tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
+    tqDebug("wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
     return -1;
   }
-  tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);
+  tqDebug("wal reader seek to ver:%"PRId64" %s", ver, id);
   return 0;
 }
 
@@ -310,26 +308,28 @@ void tqNextBlock(STqReader* pReader, SFetchRet* ret) {
         ret->fetchType = FETCH_TYPE__NONE;
         return;
       }
-      void*   body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
+
+      void*   pBody = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
       int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
       int64_t ver = pReader->pWalReader->pHead->head.version;
 
-      tqReaderSetSubmitReq2(pReader, body, bodyLen, ver);
+      tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver);
     }
 
-    while (tqNextDataBlock2(pReader)) {
+    while (tqNextDataBlock(pReader)) {
       memset(&ret->data, 0, sizeof(SSDataBlock));
       int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL);
       if (code != 0 || ret->data.info.rows == 0) {
         continue;
       }
+
       ret->fetchType = FETCH_TYPE__DATA;
       return;
     }
   }
 }
 
-int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) {
+int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) {
   pReader->msg2.msgStr = msgStr;
   pReader->msg2.msgLen = msgLen;
   pReader->msg2.ver = ver;
@@ -346,7 +346,7 @@ int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen,
   return 0;
 }
 
-bool tqNextDataBlock2(STqReader* pReader) {
+bool tqNextDataBlock(STqReader* pReader) {
   if (pReader->msg2.msgStr == NULL) {
     return false;
   }
@@ -355,13 +355,20 @@ bool tqNextDataBlock2(STqReader* pReader) {
   while (pReader->nextBlk < blockSz) {
     tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen,
             pReader->msg2.ver, pReader->nextBlk);
+
     SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
-    if (pReader->tbIdHash == NULL) return true;
+    if (pReader->tbIdHash == NULL) {
+      return true;
+    }
 
     void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t));
     if (ret != NULL) {
+      tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg2.ver, pSubmitTbData->uid);
       return true;
+    } else {
+      tqDebug("tq reader discard block, uid:%"PRId64", continue", pSubmitTbData->uid);
     }
+
     pReader->nextBlk++;
   }
 
@@ -901,7 +908,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) {
   return 0;
 }
 
-int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) {
+int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) {
   if (pReader->tbIdHash == NULL) {
     pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
     if (pReader->tbIdHash == NULL) {
@@ -910,8 +917,9 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) {
     }
   }
 
-  for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
-    int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
+  int32_t numOfTables = taosArrayGetSize(pTableUidList);
+  for (int i = 0; i < numOfTables; i++) {
+    int64_t* pKey = (int64_t*)taosArrayGet(pTableUidList, i);
     taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
   }
 
@@ -927,30 +935,34 @@ int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
   return 0;
 }
 
+// todo update the table list in wal reader
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
-  void* pIter = NULL;
+  void*   pIter = NULL;
+  int32_t vgId = TD_VID(pTq->pVnode);
+
+  // update the table list for each consumer handle
   while (1) {
     pIter = taosHashIterate(pTq->pHandle, pIter);
     if (pIter == NULL) {
       break;
     }
 
-    STqHandle* pExec = (STqHandle*)pIter;
-    if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-      int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd);
+    STqHandle* pTqHandle = (STqHandle*)pIter;
+    if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+      int32_t code = qUpdateTableListForStreamScanner(pTqHandle->execHandle.task, tbUidList, isAdd, NULL);
       if (code != 0) {
-        tqError("update qualified table error for %s", pExec->subKey);
+        tqError("update qualified table error for %s", pTqHandle->subKey);
         continue;
       }
-    } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) {
+    } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
       if (!isAdd) {
         int32_t sz = taosArrayGetSize(tbUidList);
         for (int32_t i = 0; i < sz; i++) {
           int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
-          taosHashPut(pExec->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0);
+          taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0);
         }
       }
-    } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
+    } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
       if (isAdd) {
         SArray*     qa = taosArrayInit(4, sizeof(tb_uid_t));
         SMetaReader mr = {0};
@@ -965,35 +977,50 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
           }
 
           tDecoderClear(&mr.coder);
-
-          if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pExec->execHandle.execTb.suid) {
+          if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pTqHandle->execHandle.execTb.suid) {
             tqDebug("table uid %" PRId64 " does not add to tq handle", *id);
             continue;
           }
+
           tqDebug("table uid %" PRId64 " add to tq handle", *id);
           taosArrayPush(qa, id);
         }
+
         metaReaderClear(&mr);
         if (taosArrayGetSize(qa) > 0) {
-          tqReaderAddTbUidList(pExec->execHandle.pExecReader, qa);
+          tqReaderAddTbUidList(pTqHandle->execHandle.pTqReader, qa);
         }
+
         taosArrayDestroy(qa);
       } else {
-        tqReaderRemoveTbUidList(pExec->execHandle.pExecReader, tbUidList);
+        tqReaderRemoveTbUidList(pTqHandle->execHandle.pTqReader, tbUidList);
       }
     }
   }
+
+  // update the table list handle for each stream scanner/wal reader
   while (1) {
     pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
+
     SStreamTask* pTask = *(SStreamTask**)pIter;
     if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
-      int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
+      SArray* pList = NULL;
+      int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd, pList);
       if (code != 0) {
-        tqError("update qualified table error for stream task %d", pTask->taskId);
+        tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr);
         continue;
       }
+
+      if (isAdd) { // only add qualified tables
+        tqReaderAddTbUidList(pTask->exec.pTqReader, pList);
+      } else {
+        tqReaderRemoveTbUidList(pTask->exec.pTqReader, tbUidList);
+      }
     }
   }
+
   return 0;
 }
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index b4e50312fd..5633dc37a3 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -38,7 +38,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t
 }
 
 static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) {
-  SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader->pSchemaWrapper);
+  SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper);
   if (pSW == NULL) {
     return -1;
   }
@@ -135,7 +135,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
     if (pDataBlock != NULL && pDataBlock->info.rows > 0) {
       if (pRsp->withTbName) {
         if (pOffset->type == TMQ_OFFSET__LOG) {
-          int64_t uid = pExec->pExecReader->lastBlkUid;
+          int64_t uid = pExec->pTqReader->lastBlkUid;
           if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) {
             continue;
           }
@@ -200,9 +200,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
   SArray* pSchemas = taosArrayInit(0, sizeof(void*));
 
   if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
-    STqReader* pReader = pExec->pExecReader;
-    tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver);
-    while (tqNextDataBlock2(pReader)) {
+    STqReader* pReader = pExec->pTqReader;
+    tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver);
+    while (tqNextDataBlock(pReader)) {
       taosArrayClear(pBlocks);
       taosArrayClear(pSchemas);
       SSubmitTbData* pSubmitTbDataRet = NULL;
@@ -210,7 +210,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
       }
       if (pRsp->withTbName) {
-        int64_t uid = pExec->pExecReader->lastBlkUid;
+        int64_t uid = pExec->pTqReader->lastBlkUid;
         if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
           taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
           taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper);
@@ -259,8 +259,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
       }
     }
   } else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
-    STqReader* pReader = pExec->pExecReader;
-    tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver);
+    STqReader* pReader = pExec->pTqReader;
+    tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver);
     while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) {
       taosArrayClear(pBlocks);
       taosArrayClear(pSchemas);
@@ -269,7 +269,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
       }
       if (pRsp->withTbName) {
-        int64_t uid = pExec->pExecReader->lastBlkUid;
+        int64_t uid = pExec->pTqReader->lastBlkUid;
         if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
           taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
           taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper);
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 8a89cb6bd7..22f0387d4e 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -87,7 +87,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
     return;
   }
 
-  tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz);
+  tqDebug("vgId:%d, s-task:%s write into table, block num: %d", TD_VID(pVnode), pTask->id.idStr, blockSz);
   for (int32_t i = 0; i < blockSz; i++) {
     bool         createTb = true;
     SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
@@ -382,7 +382,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
 
   int32_t blockSz = taosArrayGetSize(pBlocks);
 
-  tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz);
+  tqDebug("vgId:%d, s-task:%s write results blocks:%d into table", TD_VID(pVnode), pTask->id.idStr, blockSz);
 
   void*   pBuf = NULL;
   SArray* tagArray = NULL;
@@ -475,11 +475,9 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
           }
           for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
             SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
-            STagVal          tagVal = {
-                         .cid = pTSchema->numOfCols + step,
-                         .type = pTagData->info.type,
-            };
-            void* pData = colDataGetData(pTagData, rowId);
+
+            STagVal tagVal = {.cid = pTSchema->numOfCols + step, .type = pTagData->info.type};
+            void*   pData = colDataGetData(pTagData, rowId);
             if (colDataIsNull_s(pTagData, rowId)) {
               continue;
             } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 3d2b032156..e353988d4c 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -447,13 +447,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
 
   walApplyVer(pVnode->pWal, version);
 
-  /*vInfo("vgId:%d, push msg begin", pVnode->config.vgId);*/
   if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
     /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/
     vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
-  /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/
 
   // commit if need
   if (needCommit) {
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index b4431a7c3b..4389cc7fdf 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -127,7 +127,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
     pOperator->status = OP_NOT_OPENED;
 
     SStreamScanInfo* pInfo = pOperator->info;
-    qDebug("task stream set total blocks:%d %s", (int32_t)numOfBlocks, id);
+    qDebug("s-task set source blocks:%d %s", (int32_t)numOfBlocks, id);
     ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0);
 
     if (type == STREAM_INPUT__MERGED_SUBMIT) {
@@ -363,27 +363,28 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
   return qa;
 }
 
-int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
+int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd, SArray* pList) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  const char*    id = GET_TASKID(pTaskInfo);
+  int32_t        code = 0;
 
   if (isAdd) {
-    qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str);
+    qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id);
   }
 
   // traverse to the stream scanner node to add this table id
-  SOperatorInfo* pInfo = pTaskInfo->pRoot;
-  while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-    pInfo = pInfo->pDownstream[0];
-  }
-
-  int32_t          code = 0;
+  SOperatorInfo*   pInfo = extractOperatorInTree(pTaskInfo->pRoot, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id);
   SStreamScanInfo* pScanInfo = pInfo->info;
+
   if (isAdd) {  // add new table id
     SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
     int32_t numOfQualifiedTables = taosArrayGetSize(qa);
 
-    qDebug(" %d qualified child tables added into stream scanner", numOfQualifiedTables);
+    if (pList != NULL) {
+      taosArrayAddAll(pList, qa);
+    }
 
+    qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id);
     code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
     if (code != TSDB_CODE_SUCCESS) {
       taosArrayDestroy(qa);
@@ -424,19 +425,6 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
         }
       }
 
-#if 0
-      bool exists = false;
-      for (int32_t k = 0; k < taosArrayGetSize(pListInfo->pTableList); ++k) {
-        STableKeyInfo* pKeyInfo = taosArrayGet(pListInfo->pTableList, k);
-        if (pKeyInfo->uid == keyInfo.uid) {
-          qWarn("ignore duplicated query table uid:%" PRIu64 " added, %s", pKeyInfo->uid, pTaskInfo->id.str);
-          exists = true;
-        }
-      }
-
-      if (!exists) {
-#endif
-
       tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId);
     }
 
@@ -447,7 +435,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
 
     taosArrayDestroy(qa);
   } else {  // remove the table id in current list
-    qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
+    qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id);
     taosWLockLatch(&pTaskInfo->lock);
     code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList);
     taosWUnLockLatch(&pTaskInfo->lock);
@@ -1263,3 +1251,21 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
   destroySendMsgInfo(pSendInfo);
 }
 
+SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) {
+  SExecTaskInfo* pTaskInfo = tinfo;
+  SArray* plist = getTableListInfo(pTaskInfo);
+
+  // only extract table in the first elements
+  STableListInfo* pTableListInfo = taosArrayGetP(plist, 0);
+
+  SArray* pUidList = taosArrayInit(10, sizeof(uint64_t));
+
+  int32_t numOfTables = tableListGetSize(pTableListInfo);
+  for(int32_t i = 0; i < numOfTables; ++i) {
+    STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i);
+    taosArrayPush(pUidList, &pKeyInfo->uid);
+  }
+
+  taosArrayDestroy(plist);
+  return pUidList;
+}
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 11753c181c..52b749efee 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -2001,7 +2001,11 @@ void qStreamCloseTsdbReader(void* task) {
 }
 
 static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) {
-  if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
+  if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+    SStreamScanInfo* pScanInfo = pOperator->info;
+    STableScanInfo*  pTableScanInfo = pScanInfo->pTableScanOp->info;
+    taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo);
+  } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
     STableScanInfo* pScanInfo = pOperator->info;
     taosArrayPush(pList, &pScanInfo->base.pTableListInfo);
   } else {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 29990f2d06..21e3c75924 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1635,7 +1635,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
   if (pTaskInfo->streamInfo.submit.msgStr != NULL) {
     if (pInfo->tqReader->msg2.msgStr == NULL) {
       SPackedData submit = pTaskInfo->streamInfo.submit;
-      if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) {
+      if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) {
         qError("submit msg messed up when initing stream submit block %p", submit.msgStr);
         return NULL;
       }
@@ -1644,7 +1644,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
     blockDataCleanup(pInfo->pRes);
     SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
 
-    while (tqNextDataBlock2(pInfo->tqReader)) {
+    while (tqNextDataBlock(pInfo->tqReader)) {
       SSDataBlock block = {0};
 
       int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL);
@@ -1805,7 +1805,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
 
     /*resetTableScanInfo(pTSInfo, pWin);*/
     tsdbReaderClose(pTSInfo->base.dataReader);
-    qDebug("4");
 
     pTSInfo->base.dataReader = NULL;
     pInfo->pTableScanOp->status = OP_OPENED;
@@ -1888,7 +1887,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
     STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
     tsdbReaderClose(pTSInfo->base.dataReader);
-    qDebug("5");
 
     pTSInfo->base.dataReader = NULL;
 
@@ -1915,6 +1913,7 @@ FETCH_NEXT_BLOCK:
     if (pBlock->info.parTbName[0]) {
       streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName);
     }
+
     // TODO move into scan
     pBlock->info.calWin.skey = INT64_MIN;
     pBlock->info.calWin.ekey = INT64_MAX;
@@ -2057,7 +2056,7 @@ FETCH_NEXT_BLOCK:
 
         int32_t      current = pInfo->validBlockIndex++;
         SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current);
-        if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) {
+        if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) {
           qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current,
                  totBlockNum);
           continue;
@@ -2066,7 +2065,7 @@ FETCH_NEXT_BLOCK:
 
       blockDataCleanup(pInfo->pRes);
 
-      while (tqNextDataBlock2(pInfo->tqReader)) {
+      while (tqNextDataBlock(pInfo->tqReader)) {
         SSDataBlock block = {0};
 
         int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL);
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 1a1fb6208d..880de7d6bf 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -2432,10 +2432,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
     updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
     applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
                                     pSDataBlock->info.rows, numOfOutput);
-    SWinKey key = {
-        .ts = nextWin.skey,
-        .groupId = groupId,
-    };
+
+    SWinKey key = { .ts = nextWin.skey, .groupId = groupId };
     saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize);
     releaseOutputBuf(pInfo->pState, &key, pResult);
     if (pInfo->delKey.ts > key.ts) {
@@ -4771,6 +4769,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
       pInfo->numOfDatapack = 0;
       break;
     }
+
     pInfo->numOfDatapack++;
     printDataBlock(pBlock, "single interval recv");
 
diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h
index 66496f11f8..876b80697a 100644
--- a/source/libs/stream/inc/streamInc.h
+++ b/source/libs/stream/inc/streamInc.h
@@ -44,7 +44,7 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq*
 int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId,
                                           SEpSet* pEpSet);
 
-SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem);
+SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 361cd2cacc..59ac8a61d6 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -16,6 +16,8 @@
 #include "streamInc.h"
 #include "ttimer.h"
 
+#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 2
+
 int32_t streamInit() {
   int8_t old;
   while (1) {
@@ -96,13 +98,14 @@ int32_t streamSchedExec(SStreamTask* pTask) {
   if (schedStatus == TASK_SCHED_STATUS__INACTIVE) {
     SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
     if (pRunReq == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
       atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE);
       return -1;
     }
 
     pRunReq->head.vgId = pTask->nodeId;
-    pRunReq->streamId = pTask->streamId;
-    pRunReq->taskId = pTask->taskId;
+    pRunReq->streamId = pTask->id.streamId;
+    pRunReq->taskId = pTask->id.taskId;
 
     SRpcMsg msg = { .msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq) };
     tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg);
@@ -142,7 +145,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR
   pCont->upstreamNodeId = htonl(pReq->upstreamNodeId);
   pCont->upstreamTaskId = htonl(pReq->upstreamTaskId);
   pCont->downstreamNodeId = htonl(pTask->nodeId);
-  pCont->downstreamTaskId = htonl(pTask->taskId);
+  pCont->downstreamTaskId = htonl(pTask->id.taskId);
   pRsp->pCont = buf;
   pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
   tmsgSendRsp(pRsp);
@@ -155,7 +158,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
 
   // enqueue
   if (pData != NULL) {
-    qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->taskId, pTask->selfChildId,
+    qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId,
            pReq->srcTaskId, pReq->reqId);
 
     pData->type = STREAM_INPUT__DATA_RETRIEVE;
@@ -205,7 +208,7 @@ int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
 }
 
 int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) {
-  qDebug("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId,
+  qDebug("task %d receive dispatch req from node %d task %d", pTask->id.taskId, pReq->upstreamNodeId,
          pReq->upstreamTaskId);
 
   streamTaskEnqueue(pTask, pReq, pRsp);
@@ -228,12 +231,11 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
 
 int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) {
   ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED);
-
-  qDebug("task %d receive dispatch rsp, code: %x", pTask->taskId, code);
+  qDebug("task %d receive dispatch rsp, code: %x", pTask->id.taskId, code);
 
   if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
     int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1);
-    qDebug("task %d is shuffle, left waiting rsp %d", pTask->taskId, leftRsp);
+    qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp);
     if (leftRsp > 0) return 0;
   }
 
@@ -261,7 +263,7 @@ int32_t streamProcessRunReq(SStreamTask* pTask) {
 }
 
 int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
-  qDebug("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId);
+  qDebug("task %d receive retrieve req from node %d task %d", pTask->id.taskId, pReq->srcNodeId, pReq->srcTaskId);
 
   streamTaskEnqueueRetrieve(pTask, pReq, pRsp);
 
@@ -275,26 +277,43 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S
   return 0;
 }
 
+bool tInputQueueIsFull(const SStreamTask* pTask) {
+  return taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUEU_CAPACITY;
+}
+
 int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) {
   int8_t type = pItem->type;
 
   if (type == STREAM_INPUT__DATA_SUBMIT) {
     SStreamDataSubmit2* pSubmitBlock = streamSubmitBlockClone((SStreamDataSubmit2*)pItem);
     if (pSubmitBlock == NULL) {
-      qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask);
+      qDebug("task %d %p submit enqueue failed since out of memory", pTask->id.taskId, pTask);
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
       return -1;
     }
 
     int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1;
-    qDebug("stream task:%d %p submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->taskId,
-           pTask, pItem, pSubmitBlock, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen,
+    qDebug("s-task:%s submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->id.idStr,
+           pItem, pSubmitBlock, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen,
            pSubmitBlock->submit.ver, total);
 
+    if (total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) {
+      qDebug("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY);
+      streamDataSubmitDestroy(pSubmitBlock);
+      return -1;
+    }
+
     taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock);
   } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE ||
              type == STREAM_INPUT__REF_DATA_BLOCK) {
+    int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1;
+    if (total > 2) {
+      qDebug("stream task input queue is full, abort");
+      return -1;
+    }
+
+    qDebug("s-task:%s data block enqueue, total in queue:%d", pTask->id.idStr, total);
     taosWriteQitem(pTask->inputQueue->queue, pItem);
   } else if (type == STREAM_INPUT__CHECKPOINT) {
     taosWriteQitem(pTask->inputQueue->queue, pItem);
diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c
index 3fba1cb556..63d15f134d 100644
--- a/source/libs/stream/src/streamData.c
+++ b/source/libs/stream/src/streamData.c
@@ -67,7 +67,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
   return 0;
 }
 
-SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) {
+SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type) {
   SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0);
 
   if (pDataSubmit == NULL) {
@@ -82,7 +82,7 @@ SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) {
 
   pDataSubmit->submit = submit;
   *pDataSubmit->dataRef = 1;   // initialize the reference count to be 1
-  pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT;
+  pDataSubmit->type = type;
 
   return pDataSubmit;
 }
@@ -139,28 +139,27 @@ SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit) {
   return pSubmitClone;
 }
 
-SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) {
-  ASSERT(elem);
-  if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) {
+SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) {
+  if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) {
     SStreamDataBlock* pBlock = (SStreamDataBlock*)dst;
-    SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)elem;
+    SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem;
     taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks);
     taosArrayDestroy(pBlockSrc->blocks);
-    taosFreeQitem(elem);
+    taosFreeQitem(pElem);
     return dst;
-  } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) {
+  } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
     SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst;
-    SStreamDataSubmit2*   pBlockSrc = (SStreamDataSubmit2*)elem;
+    SStreamDataSubmit2*   pBlockSrc = (SStreamDataSubmit2*)pElem;
     streamMergeSubmit(pMerged, pBlockSrc);
-    taosFreeQitem(elem);
+    taosFreeQitem(pElem);
     return dst;
-  } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) {
+  } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
     SStreamMergedSubmit2* pMerged = streamMergedSubmitNew();
     ASSERT(pMerged);
     streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst);
-    streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem);
+    streamMergeSubmit(pMerged, (SStreamDataSubmit2*)pElem);
     taosFreeQitem(dst);
-    taosFreeQitem(elem);
+    taosFreeQitem(pElem);
     return (SStreamQueueItem*)pMerged;
   } else {
     return NULL;
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 7e7c23f98a..4e491f906a 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -121,9 +121,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
   int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
 
   SStreamRetrieveReq req = {
-      .streamId = pTask->streamId,
+      .streamId = pTask->id.streamId,
       .srcNodeId = pTask->nodeId,
-      .srcTaskId = pTask->taskId,
+      .srcTaskId = pTask->id.taskId,
       .pRetrieve = pRetrieve,
       .retrieveLen = dataStrLen,
   };
@@ -168,7 +168,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
     }
     buf = NULL;
 
-    qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->taskId,
+    qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->id.idStr,
            pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId);
   }
   code = 0;
@@ -238,7 +238,7 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq*
   msg.pCont = buf;
   msg.msgType = TDMT_STREAM_TASK_CHECK;
 
-  qDebug("dispatch from task %d to task %d node %d: check msg", pTask->taskId, pReq->downstreamTaskId, nodeId);
+  qDebug("dispatch from task %d to task %d node %d: check msg", pTask->id.taskId, pReq->downstreamTaskId, nodeId);
 
   tmsgSendReq(pEpSet, &msg);
 
@@ -282,7 +282,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov
 
   tmsgSendReq(pEpSet, &msg);
 
-  qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->taskId, pReq->taskId, vgId);
+  qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->id.taskId, pReq->taskId, vgId);
 
   return 0;
 FAIL:
@@ -319,7 +319,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p
   msg.pCont = buf;
   msg.msgType = pTask->dispatchMsgType;
 
-  qDebug("dispatch from task %d to task %d node %d: data msg", pTask->taskId, pReq->taskId, vgId);
+  qDebug("dispatch from task %d to task %d node %d: data msg", pTask->id.taskId, pReq->taskId, vgId);
 
   tmsgSendReq(pEpSet, &msg);
 
@@ -382,9 +382,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
 
   if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
     SStreamDispatchReq req = {
-        .streamId = pTask->streamId,
+        .streamId = pTask->id.streamId,
         .dataSrcVgId = pData->srcVgId,
-        .upstreamTaskId = pTask->taskId,
+        .upstreamTaskId = pTask->id.taskId,
         .upstreamChildId = pTask->selfChildId,
         .upstreamNodeId = pTask->nodeId,
         .blockNum = blockNum,
@@ -408,7 +408,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
 
     req.taskId = downstreamTaskId;
 
-    qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId,
+    qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->id.taskId, pTask->selfChildId,
            downstreamTaskId, vgId);
 
     if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) {
@@ -432,9 +432,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
     }
 
     for (int32_t i = 0; i < vgSz; i++) {
-      pReqs[i].streamId = pTask->streamId;
+      pReqs[i].streamId = pTask->id.streamId;
       pReqs[i].dataSrcVgId = pData->srcVgId;
-      pReqs[i].upstreamTaskId = pTask->taskId;
+      pReqs[i].upstreamTaskId = pTask->id.taskId;
       pReqs[i].upstreamChildId = pTask->selfChildId;
       pReqs[i].upstreamNodeId = pTask->nodeId;
       pReqs[i].blockNum = 0;
@@ -503,13 +503,13 @@ int32_t streamDispatch(SStreamTask* pTask) {
 
   SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue);
   if (pBlock == NULL) {
-    qDebug("stream stop dispatching since no output: task %d", pTask->taskId);
+    qDebug("stream stop dispatching since no output: task %d", pTask->id.taskId);
     atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
     return 0;
   }
   ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
 
-  qDebug("stream dispatching: task %d", pTask->taskId);
+  qDebug("stream dispatching: task %d", pTask->id.taskId);
 
   int32_t code = 0;
   if (streamDispatchAllBlocks(pTask, pBlock) < 0) {
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 6ef327049c..d23590a08b 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -18,8 +18,9 @@
 #define STREAM_EXEC_MAX_BATCH_NUM 100
 
 static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
-  int32_t code;
-  void*   exec = pTask->exec.executor;
+  int32_t code = TSDB_CODE_SUCCESS;
+  void*   pExecutor = pTask->exec.pExecutor;
+
   while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
     qError("stream task wait for the end of fill history");
     taosMsleep(2);
@@ -30,31 +31,35 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
   const SStreamQueueItem* pItem = (const SStreamQueueItem*)data;
   if (pItem->type == STREAM_INPUT__GET_RES) {
     const SStreamTrigger* pTrigger = (const SStreamTrigger*)data;
-    qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
+    qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
   } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
     ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
     const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data;
-    qDebug("stream task:%d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr,
+    qDebug("s-task:%s set submit blocks as input %p %p %d ver:%" PRId64, pTask->id.idStr, pSubmit, pSubmit->submit.msgStr,
            pSubmit->submit.msgLen, pSubmit->submit.ver);
-    qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
+    qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
   } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
     const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data;
-    SArray*                 blocks = pBlock->blocks;
-    qDebug("task %d %p set ssdata input", pTask->taskId, pTask);
-    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK);
+
+    SArray* pBlockList = pBlock->blocks;
+    int32_t numOfBlocks = taosArrayGetSize(pBlockList);
+    qDebug("s-task:%s set sdata blocks as input num:%d, ver:%"PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer);
+    qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK);
   } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) {
     const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data;
-    SArray*                     blocks = pMerged->submits;
-    qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size);
-    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT);
+
+    SArray* pBlockList = pMerged->submits;
+    int32_t numOfBlocks = taosArrayGetSize(pBlockList);
+    qDebug("st-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks);
+    qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
   } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
     const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data;
-    qSetMultiStreamInput(exec, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
+    qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
   } else {
     ASSERT(0);
   }
 
-  // exec
+  // pExecutor
   while (1) {
     if (pTask->taskStatus == TASK_STATUS__DROPPING) {
       return 0;
@@ -62,26 +67,28 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
 
     SSDataBlock* output = NULL;
     uint64_t     ts = 0;
-    if ((code = qExecTask(exec, &output, &ts)) < 0) {
+    if ((code = qExecTask(pExecutor, &output, &ts)) < 0) {
       if (code == TSDB_CODE_QRY_IN_EXEC) {
-        resetTaskInfo(exec);
+        resetTaskInfo(pExecutor);
       }
-      /*ASSERT(false);*/
-      qError("unexpected stream execution, stream %" PRId64 " task: %d,  since %s", pTask->streamId, pTask->taskId,
-             terrstr());
+
+      qError("unexpected stream execution, s-task:%s since %s", pTask->id.idStr, terrstr());
       continue;
     }
+
     if (output == NULL) {
       if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
-        SSDataBlock             block = {0};
+        SSDataBlock block = {0};
+
         const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data;
         ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
+
         assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
         block.info.type = STREAM_PULL_OVER;
         block.info.childId = pTask->selfChildId;
         taosArrayPush(pRes, &block);
 
-        qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->taskId, pTask->selfChildId,
+        qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId,
                pRetrieveBlock->reqId);
       }
       break;
@@ -94,20 +101,21 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
       continue;
     }
 
-    qDebug("task %d(child %d) executed and get block", pTask->taskId, pTask->selfChildId);
+    qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId);
 
     SSDataBlock block = {0};
     assignOneDataBlock(&block, output);
     block.info.childId = pTask->selfChildId;
     taosArrayPush(pRes, &block);
   }
+
   return 0;
 }
 
 int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
   ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
 
-  void* exec = pTask->exec.executor;
+  void* exec = pTask->exec.pExecutor;
 
   qSetStreamOpOpen(exec);
   bool finished = false;
@@ -147,17 +155,17 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
 
       batchCnt++;
 
-      qDebug("task %d scan exec block num %d, block limit %d", pTask->taskId, batchCnt, batchSz);
+      qDebug("task %d scan exec block num %d, block limit %d", pTask->id.taskId, batchCnt, batchSz);
 
       if (batchCnt >= batchSz) break;
     }
     if (taosArrayGetSize(pRes) == 0) {
       if (finished) {
         taosArrayDestroy(pRes);
-        qDebug("task %d finish recover exec task ", pTask->taskId);
+        qDebug("task %d finish recover exec task ", pTask->id.taskId);
         break;
       } else {
-        qDebug("task %d continue recover exec task ", pTask->taskId);
+        qDebug("task %d continue recover exec task ", pTask->id.taskId);
         continue;
       }
     }
@@ -173,7 +181,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
     streamTaskOutput(pTask, qRes);
 
     if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
-      qDebug("task %d scan exec dispatch block num %d", pTask->taskId, batchCnt);
+      qDebug("task %d scan exec dispatch block num %d", pTask->id.taskId, batchCnt);
       streamDispatch(pTask);
     }
     if (finished) break;
@@ -186,7 +194,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) {
   // fetch all queue item, merge according to batchLimit
   int32_t numOfItems = taosReadAllQitems(pTask->inputQueue1, pTask->inputQall);
   if (numOfItems == 0) {
-    qDebug("task: %d, stream task exec over, queue empty", pTask->taskId);
+    qDebug("task: %d, stream task exec over, queue empty", pTask->id.taskId);
     return 0;
   }
   SStreamQueueItem* pMerged = NULL;
@@ -221,30 +229,33 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) {
 
 int32_t streamExecForAll(SStreamTask* pTask) {
   while (1) {
-    int32_t batchCnt = 1;
-    void*   input = NULL;
+    int32_t batchSize = 1;
+    void*   pInput = NULL;
+
+    // merge multiple input data if possible in the input queue.
     while (1) {
       SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
       if (qItem == NULL) {
-        qDebug("stream task exec over, queue empty, task: %d", pTask->taskId);
+        qDebug("stream task exec over, queue empty, task: %d", pTask->id.taskId);
         break;
       }
-      if (input == NULL) {
-        input = qItem;
+
+      if (pInput == NULL) {
+        pInput = qItem;
         streamQueueProcessSuccess(pTask->inputQueue);
         if (pTask->taskLevel == TASK_LEVEL__SINK) {
           break;
         }
       } else {
-        void* newRet;
-        if ((newRet = streamMergeQueueItem(input, qItem)) == NULL) {
+        void* newRet = NULL;
+        if ((newRet = streamMergeQueueItem(pInput, qItem)) == NULL) {
           streamQueueProcessFail(pTask->inputQueue);
           break;
         } else {
-          batchCnt++;
-          input = newRet;
+          batchSize++;
+          pInput = newRet;
           streamQueueProcessSuccess(pTask->inputQueue);
-          if (batchCnt > STREAM_EXEC_MAX_BATCH_NUM) {
+          if (batchSize > STREAM_EXEC_MAX_BATCH_NUM) {
             break;
           }
         }
@@ -252,75 +263,82 @@ int32_t streamExecForAll(SStreamTask* pTask) {
     }
 
     if (pTask->taskStatus == TASK_STATUS__DROPPING) {
-      if (input) streamFreeQitem(input);
+      if (pInput) {
+        streamFreeQitem(pInput);
+      }
       return 0;
     }
 
-    if (input == NULL) {
+    if (pInput == NULL) {
       break;
     }
 
     if (pTask->taskLevel == TASK_LEVEL__SINK) {
-      ASSERT(((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_BLOCK);
-      streamTaskOutput(pTask, input);
+      ASSERT(((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_BLOCK);
+      streamTaskOutput(pTask, pInput);
       continue;
     }
 
     SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
+    qDebug("s-task:%s exec begin, msg batch: %d", pTask->id.idStr, batchSize);
 
-    qDebug("stream task:%d exec begin, msg batch: %d", pTask->taskId, batchCnt);
-    streamTaskExecImpl(pTask, input, pRes);
+    streamTaskExecImpl(pTask, pInput, pRes);
 
-    qDebug("stream task:%d exec end", pTask->taskId);
+    qDebug("s-task:%s exec end", pTask->id.idStr);
 
     if (taosArrayGetSize(pRes) != 0) {
       SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
       if (qRes == NULL) {
         taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
-        streamFreeQitem(input);
+        streamFreeQitem(pInput);
         return -1;
       }
+
       qRes->type = STREAM_INPUT__DATA_BLOCK;
       qRes->blocks = pRes;
 
-      if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) {
-        SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input;
+      if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_SUBMIT) {
+        SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)pInput;
         qRes->childId = pTask->selfChildId;
         qRes->sourceVer = pSubmit->ver;
-      } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) {
-        SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input;
+      } else if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__MERGED_SUBMIT) {
+        SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)pInput;
         qRes->childId = pTask->selfChildId;
         qRes->sourceVer = pMerged->ver;
       }
 
       if (streamTaskOutput(pTask, qRes) < 0) {
         taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
-        streamFreeQitem(input);
+        streamFreeQitem(pInput);
         taosFreeQitem(qRes);
         return -1;
       }
     } else {
       taosArrayDestroy(pRes);
     }
-    streamFreeQitem(input);
+    streamFreeQitem(pInput);
   }
   return 0;
 }
 
 int32_t streamTryExec(SStreamTask* pTask) {
+  // this function may be executed by multi-threads, so status check is required.
   int8_t schedStatus =
       atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE);
+
   if (schedStatus == TASK_SCHED_STATUS__WAITING) {
     int32_t code = streamExecForAll(pTask);
     if (code < 0) {
       atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__FAILED);
       return -1;
     }
+
     atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE);
 
     if (!taosQueueEmpty(pTask->inputQueue->queue)) {
       streamSchedExec(pTask);
     }
   }
+
   return 0;
 }
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 66d98e90bf..577f6d6e00 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -23,6 +23,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
+
   int32_t len = strlen(path) + 20;
   char*   streamPath = taosMemoryCalloc(1, len);
   sprintf(streamPath, "%s/%s", path, "stream");
@@ -83,7 +84,7 @@ void streamMetaClose(SStreamMeta* pMeta) {
       taosTmrStop(pTask->timer);
       pTask->timer = NULL;
     }
-    tFreeSStreamTask(pTask);
+    tFreeStreamTask(pTask);
     /*streamMetaReleaseTask(pMeta, pTask);*/
   }
   taosHashCleanup(pMeta->pTasks);
@@ -111,12 +112,12 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg,
     goto FAIL;
   }
 
-  if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
+  if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
     goto FAIL;
   }
 
-  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) {
-    taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t));
+  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) {
+    taosHashRemove(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t));
     ASSERT(0);
     goto FAIL;
   }
@@ -124,7 +125,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg,
   return 0;
 
 FAIL:
-  if (pTask) tFreeSStreamTask(pTask);
+  if (pTask) tFreeStreamTask(pTask);
   return -1;
 }
 #endif
@@ -147,7 +148,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   tEncodeSStreamTask(&encoder, pTask);
   tEncoderClear(&encoder);
 
-  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) {
+  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) {
     return -1;
   }
 
@@ -165,7 +166,7 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
     return -1;
   }
 
-  taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*));
+  taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*));
 
   return 0;
 }
@@ -207,7 +208,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   ASSERT(left >= 0);
   if (left == 0) {
     ASSERT(atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING);
-    tFreeSStreamTask(pTask);
+    tFreeStreamTask(pTask);
   }
 }
 
@@ -297,7 +298,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
       return -1;
     }
 
-    if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
+    if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
       tdbFree(pKey);
       tdbFree(pVal);
       tdbTbcClose(pCur);
diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c
index 87058bf490..3e7a02b8d5 100644
--- a/source/libs/stream/src/streamRecover.c
+++ b/source/libs/stream/src/streamRecover.c
@@ -16,7 +16,7 @@
 #include "streamInc.h"
 
 int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
-  qDebug("task %d at node %d launch recover", pTask->taskId, pTask->nodeId);
+  qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId);
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
     atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE);
     streamSetParamForRecover(pTask);
@@ -56,8 +56,8 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
 // checkstatus
 int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
   SStreamTaskCheckReq req = {
-      .streamId = pTask->streamId,
-      .upstreamTaskId = pTask->taskId,
+      .streamId = pTask->id.streamId,
+      .upstreamTaskId = pTask->id.taskId,
       .upstreamNodeId = pTask->nodeId,
       .childId = pTask->selfChildId,
   };
@@ -68,7 +68,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
     req.downstreamTaskId = pTask->fixedEpDispatcher.taskId;
     pTask->checkReqId = req.reqId;
 
-    qDebug("task %d at node %d check downstream task %d at node %d", pTask->taskId, pTask->nodeId, req.downstreamTaskId,
+    qDebug("task %d at node %d check downstream task %d at node %d", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId,
            req.downstreamNodeId);
     streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet);
   } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
@@ -83,12 +83,12 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
       taosArrayPush(pTask->checkReqIds, &req.reqId);
       req.downstreamNodeId = pVgInfo->vgId;
       req.downstreamTaskId = pVgInfo->taskId;
-      qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->taskId, pTask->nodeId,
+      qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->id.taskId, pTask->nodeId,
              req.downstreamTaskId, req.downstreamNodeId);
       streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
     }
   } else {
-    qDebug("task %d at node %d direct launch recover since no downstream", pTask->taskId, pTask->nodeId);
+    qDebug("task %d at node %d direct launch recover since no downstream", pTask->id.taskId, pTask->nodeId);
     streamTaskLaunchRecover(pTask, version);
   }
   return 0;
@@ -104,7 +104,7 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp
       .downstreamNodeId = pRsp->downstreamNodeId,
       .childId = pRsp->childId,
   };
-  qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->taskId, pTask->nodeId,
+  qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->id.taskId, pTask->nodeId,
          req.downstreamTaskId, req.downstreamNodeId);
   if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
     streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet);
@@ -160,11 +160,11 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp*
 
 // common
 int32_t streamSetParamForRecover(SStreamTask* pTask) {
-  void* exec = pTask->exec.executor;
+  void* exec = pTask->exec.pExecutor;
   return qStreamSetParamForRecover(exec);
 }
 int32_t streamRestoreParam(SStreamTask* pTask) {
-  void* exec = pTask->exec.executor;
+  void* exec = pTask->exec.pExecutor;
   return qStreamRestoreParam(exec);
 }
 int32_t streamSetStatusNormal(SStreamTask* pTask) {
@@ -174,14 +174,14 @@ int32_t streamSetStatusNormal(SStreamTask* pTask) {
 
 // source
 int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) {
-  void* exec = pTask->exec.executor;
+  void* exec = pTask->exec.pExecutor;
   return qStreamSourceRecoverStep1(exec, ver);
 }
 
 int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) {
   pReq->msgHead.vgId = pTask->nodeId;
-  pReq->streamId = pTask->streamId;
-  pReq->taskId = pTask->taskId;
+  pReq->streamId = pTask->id.streamId;
+  pReq->taskId = pTask->id.taskId;
   return 0;
 }
 
@@ -192,13 +192,13 @@ int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) {
 
 int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) {
   pReq->msgHead.vgId = pTask->nodeId;
-  pReq->streamId = pTask->streamId;
-  pReq->taskId = pTask->taskId;
+  pReq->streamId = pTask->id.streamId;
+  pReq->taskId = pTask->id.taskId;
   return 0;
 }
 
 int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) {
-  void* exec = pTask->exec.executor;
+  void* exec = pTask->exec.pExecutor;
   if (qStreamSourceRecoverStep2(exec, ver) < 0) {
   }
   return streamScanExec(pTask, 100);
@@ -206,7 +206,7 @@ int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) {
 
 int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) {
   SStreamRecoverFinishReq req = {
-      .streamId = pTask->streamId,
+      .streamId = pTask->id.streamId,
       .childId = pTask->selfChildId,
   };
   // serialize
@@ -227,13 +227,13 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) {
 
 // agg
 int32_t streamAggRecoverPrepare(SStreamTask* pTask) {
-  void* exec = pTask->exec.executor;
+  void* exec = pTask->exec.pExecutor;
   pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo);
   return 0;
 }
 
 int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) {
-  void* exec = pTask->exec.executor;
+  void* exec = pTask->exec.pExecutor;
   if (qStreamRestoreParam(exec) < 0) {
     return -1;
   }
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 411726075e..04a1414438 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -121,7 +121,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
 
   char statePath[1024];
   if (!specPath) {
-    sprintf(statePath, "%s/%d", path, pTask->taskId);
+    sprintf(statePath, "%s/%d", path, pTask->id.taskId);
   } else {
     memset(statePath, 0, 1024);
     tstrncpy(statePath, path, 1024);
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index e9aba0bc39..c1f50178dd 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -21,8 +21,14 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
   if (pTask == NULL) {
     return NULL;
   }
-  pTask->taskId = tGenIdPI32();
-  pTask->streamId = streamId;
+
+  pTask->id.taskId = tGenIdPI32();
+  pTask->id.streamId = streamId;
+
+  char buf[128] = {0};
+  sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId);
+
+  pTask->id.idStr = taosStrdup(buf);
   pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
   pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
@@ -50,8 +56,8 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
 
 int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   if (tStartEncode(pEncoder) < 0) return -1;
-  if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1;
@@ -103,8 +109,8 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
 
 int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   if (tStartDecode(pDecoder) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1;
@@ -162,24 +168,43 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   return 0;
 }
 
-void tFreeSStreamTask(SStreamTask* pTask) {
-  qDebug("free stream task %d", pTask->taskId);
-  if (pTask->inputQueue) streamQueueClose(pTask->inputQueue);
-  if (pTask->outputQueue) streamQueueClose(pTask->outputQueue);
-  if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
-  if (pTask->exec.executor) qDestroyTask(pTask->exec.executor);
+void tFreeStreamTask(SStreamTask* pTask) {
+  qDebug("free s-task:%s", pTask->id.idStr);
+
+  if (pTask->inputQueue) {
+    streamQueueClose(pTask->inputQueue);
+  }
+  if (pTask->outputQueue) {
+    streamQueueClose(pTask->outputQueue);
+  }
+  if (pTask->exec.qmsg) {
+    taosMemoryFree(pTask->exec.qmsg);
+  }
+
+  if (pTask->exec.pExecutor) {
+    qDestroyTask(pTask->exec.pExecutor);
+    pTask->exec.pExecutor = NULL;
+  }
+
+  if (pTask->exec.pTqReader != NULL) {
+    pTask->exec.pTqReader = NULL;
+  }
+
   taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree);
   if (pTask->outputType == TASK_OUTPUT__TABLE) {
     tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper);
     taosMemoryFree(pTask->tbSink.pTSchema);
   }
+
   if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
     taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos);
     taosArrayDestroy(pTask->checkReqIds);
     pTask->checkReqIds = NULL;
   }
 
-  if (pTask->pState) streamStateClose(pTask->pState);
+  if (pTask->pState) {
+    streamStateClose(pTask->pState);
+  }
 
   taosMemoryFree(pTask);
 }

From 7419c0bfe99cc6802ebe9ef9e94ad6829755f09a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 10 Apr 2023 16:56:14 +0800
Subject: [PATCH 02/65] enh(stream): the stream will start after vnode restore
 being completed.

---
 include/libs/stream/tstream.h           |   3 +-
 include/libs/wal/wal.h                  |   1 +
 source/dnode/mnode/impl/src/mndStream.c |   2 +-
 source/dnode/vnode/CMakeLists.txt       |   2 +
 source/dnode/vnode/src/inc/tq.h         |  14 ++-
 source/dnode/vnode/src/inc/vnodeInt.h   |   5 +-
 source/dnode/vnode/src/tq/tq.c          | 113 ++++++++++----------
 source/dnode/vnode/src/tq/tqPush.c      |  11 +-
 source/dnode/vnode/src/tq/tqRestore.c   | 132 ++++++++++++++++++++++++
 source/dnode/vnode/src/tq/tqUtil.c      |  72 +++++++++++++
 source/dnode/vnode/src/vnd/vnodeSync.c  |   3 +
 source/libs/stream/src/streamMeta.c     |  18 +++-
 source/libs/wal/src/walRead.c           |   2 +
 13 files changed, 300 insertions(+), 78 deletions(-)
 create mode 100644 source/dnode/vnode/src/tq/tqRestore.c
 create mode 100644 source/dnode/vnode/src/tq/tqUtil.c

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index d9b82c8c59..a338413502 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -50,6 +50,7 @@ enum {
   TASK_STATUS__RECOVER_PREPARE,
   TASK_STATUS__RECOVER1,
   TASK_STATUS__RECOVER2,
+  TASK_STATUS_RESTORE,  // only available for source task to replay WAL from the checkpoint
 };
 
 enum {
@@ -576,7 +577,7 @@ typedef struct SStreamMeta {
   TTB*         pTaskDb;
   TTB*         pCheckpointDb;
   SHashObj*    pTasks;
-  SHashObj*    pRecoverStatus;
+  SHashObj*    pRestoreTasks;
   void*        ahandle;
   TXN*         txn;
   FTaskExpand* expandFunc;
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index fdd21c7092..0a359bfd42 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -197,6 +197,7 @@ void        walReadReset(SWalReader *pReader);
 int32_t     walReadVer(SWalReader *pRead, int64_t ver);
 int32_t     walReadSeekVer(SWalReader *pRead, int64_t ver);
 int32_t     walNextValidMsg(SWalReader *pRead);
+int64_t     walReaderGetCurrentVer(const SWalReader* pReader);
 
 // only for tq usage
 void    walSetReaderCapacity(SWalReader *pRead, int32_t capacity);
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index ab83f29ef9..a0118ee749 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -35,7 +35,7 @@
 
 static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
-static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
+static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream);
 static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 9911752f8e..c713d1e247 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -57,6 +57,7 @@ target_sources(
 
     # tq
     "src/tq/tq.c"
+    "src/tq/tqUtil.c"
     "src/tq/tqScan.c"
     "src/tq/tqMeta.c"
     "src/tq/tqRead.c"
@@ -64,6 +65,7 @@ target_sources(
     "src/tq/tqPush.c"
     "src/tq/tqSink.c"
     "src/tq/tqCommit.c"
+    "src/tq/tqRestore.c"
     "src/tq/tqSnapshot.c"
     "src/tq/tqOffsetSnapshot.c"
 )
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index 884c01d397..d4af9ac481 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -128,6 +128,10 @@ typedef struct {
   tmr_h  timer;
 } STqMgmt;
 
+typedef struct {
+  int32_t size;
+} STqOffsetHead;
+
 static STqMgmt tqMgmt = {0};
 
 int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle);
@@ -154,10 +158,6 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_
 int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
 int32_t tqMetaRestoreCheckInfo(STQ* pTq);
 
-typedef struct {
-  int32_t size;
-} STqOffsetHead;
-
 STqOffsetStore* tqOffsetOpen(STQ* pTq);
 void            tqOffsetClose(STqOffsetStore*);
 STqOffset*      tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey);
@@ -176,6 +176,12 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
 
 // tqStream
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
+int32_t tqDoRestoreSourceStreamTasks(STQ* pTq);
+
+// tq util
+void    createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId);
+int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver);
+int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 8b01ba237f..412c2549b5 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -192,8 +192,9 @@ void    tqCleanUp();
 STQ*    tqOpen(const char* path, SVnode* pVnode);
 void    tqClose(STQ*);
 int     tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
-int     tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type);
-int     tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
+int     tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type);
+int     tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
+int     tqRestoreStreamTasks(STQ* pTq);    // restore all stream tasks after vnode launching completed.
 
 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 44d5e26603..aa8960e977 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -16,6 +16,7 @@
 #include "tq.h"
 
 #define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)
+#define ALL_STREAM_TASKS_ID       (-1)
 
 int32_t tqInit() {
   int8_t old;
@@ -85,21 +86,6 @@ static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight)
          pLeft->val.version <= pRight->val.version;
 }
 
-// stream_task:stream_id:task_id
-static void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
-  int32_t n = 12;
-  char* p = dst;
-
-  memcpy(p, "stream_task:", n);
-  p += n;
-
-  int32_t inc = tintToHex(streamId, p);
-  p += inc;
-
-  *(p++) = ':';
-  tintToHex(taskId, p);
-}
-
 STQ* tqOpen(const char* path, SVnode* pVnode) {
   STQ* pTq = taosMemoryCalloc(1, sizeof(STQ));
   if (pTq == NULL) {
@@ -470,7 +456,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
   // till now, all data has been transferred to consumer, new data needs to push client once arrived.
   if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
       dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
-    code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
+    code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
     taosWUnLockLatch(&pTq->lock);
     return code;
   }
@@ -880,7 +866,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
       atomic_store_32(&pHandle->epoch, -1);
 
       // remove if it has been register in the push manager, and return one empty block to consumer
-      tqUnregisterPushEntry(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true);
+      tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true);
 
       atomic_store_64(&pHandle->consumerId, req.newConsumerId);
       atomic_add_fetch_32(&pHandle->epoch, 1);
@@ -925,6 +911,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   // expand executor
   if (pTask->fillHistory) {
     pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
+  } else {
+    pTask->taskStatus = TASK_STATUS_RESTORE;
   }
 
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
@@ -1382,21 +1370,6 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   return 0;
 }
 
-static int32_t doAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
-  int32_t code = tAppendDataForStream(pTask, pQueueItem);
-  if (code < 0) {
-    tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver);
-    return -1;
-  }
-
-  if (streamSchedExec(pTask) < 0) {
-    tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno));
-    return -1;
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
 static void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
   STqOffset offset = {0};
   tqOffsetResetToLog(&offset.val, ver);
@@ -1410,7 +1383,7 @@ static void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int
 static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit,
                                          const char* key, int64_t ver) {
   doSaveTaskOffset(pOffsetStore, key, ver);
-  int32_t code = doAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver);
+  int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver);
 
   // remove the offset, if all functions are completed successfully.
   if (code == TSDB_CODE_SUCCESS) {
@@ -1504,33 +1477,15 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
       // all data has been retrieved from WAL, let's try submit block directly.
       if (code == TSDB_CODE_SUCCESS) {  // all data retrieved, abort
         // append the data for the stream
-        SFetchRet ret = {0};
+        SFetchRet ret = {.data.info.type = STREAM_NORMAL};
         terrno = 0;
 
         tqNextBlock(pTask->exec.pTqReader, &ret);
         if (ret.fetchType == FETCH_TYPE__DATA) {
-          SStreamDataBlock* pBlocks = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
-          if (pBlocks == NULL) { // failed, do nothing
-            terrno = TSDB_CODE_OUT_OF_MEMORY;
+          code = launchTaskForWalBlock(pTask, &ret, pOffset);
+          if (code != TSDB_CODE_SUCCESS) {
             continue;
           }
-
-          ret.data.info.type = STREAM_NORMAL;
-          pBlocks->type = STREAM_INPUT__DATA_BLOCK;
-          pBlocks->sourceVer = pOffset->val.version;
-          pBlocks->blocks = taosArrayInit(0, sizeof(SSDataBlock));
-          taosArrayPush(pBlocks->blocks, &ret.data);
-
-          int64_t* ts = (int64_t*)(((SColumnInfoData*)ret.data.pDataBlock->pData)->pData);
-//          tqDebug("-----------%ld\n", ts[0]);
-
-          code = doAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pBlocks, pBlocks->sourceVer);
-          if (code == TSDB_CODE_SUCCESS) {
-            pOffset->val.version = pTask->exec.pTqReader->pWalReader->curVersion;
-            tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
-                    pOffset->val.version);
-          }
-
         } else {  // FETCH_TYPE__NONE, let's try submit block directly
           tqDebug("s-task:%s data in WAL are all consumed, try data in submit message", pTask->id.idStr);
           addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
@@ -1555,15 +1510,29 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
 
 int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   SStreamTaskRunReq* pReq = pMsg->pCont;
-  int32_t            taskId = pReq->taskId;
-  SStreamTask*       pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
-  if (pTask) {
-    tqDebug("stream task:%d start to process run req", pTask->id.taskId);
-    streamProcessRunReq(pTask);
-    streamMetaReleaseTask(pTq->pStreamMeta, pTask);
+
+  int32_t taskId = pReq->taskId;
+  int32_t vgId = TD_VID(pTq->pVnode);
+
+  if (taskId == ALL_STREAM_TASKS_ID) {  // all tasks are restored from the wal
+    tqDoRestoreSourceStreamTasks(pTq);
     return 0;
   } else {
-    return -1;
+    SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
+    if (pTask != NULL) {
+      if (pTask->taskStatus == TASK_STATUS__NORMAL) {
+        tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
+        streamProcessRunReq(pTask);
+      } else {
+        tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr);
+      }
+
+      streamMetaReleaseTask(pTq->pStreamMeta, pTask);
+      return 0;
+    } else {
+      tqError("vgId:%d failed to found s-task, taskId:%d", vgId, taskId);
+      return -1;
+    }
   }
 }
 
@@ -1703,3 +1672,25 @@ FAIL:
 }
 
 int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; }
+
+int32_t tqRestoreStreamTasks(STQ* pTq) {
+  int32_t vgId = TD_VID(pTq->pVnode);
+
+  SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
+  if (pRunReq == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr(terrno));
+    return -1;
+  }
+
+  tqInfo("vgId:%d start to restore all stream tasks", vgId);
+  
+  pRunReq->head.vgId = vgId;
+  pRunReq->streamId = 0;
+  pRunReq->taskId = ALL_STREAM_TASKS_ID;
+
+  SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
+  tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg);
+
+  return 0;
+}
diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c
index dd003aec98..d651e945b5 100644
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@@ -322,8 +322,11 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
     taosWUnLockLatch(&pTq->lock);
   }
 
-  // push data for stream processing
-  if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) {
+  // push data for stream processing:
+  // 1. the vnode isn't in the restore procedure.
+  // 2. the vnode should be the leader.
+  // 3. the stream is not suspended yet.
+  if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && (!pTq->pVnode->restored)) {
     if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) {
       return 0;
     }
@@ -352,7 +355,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
   return 0;
 }
 
-int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg,
+int32_t tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg,
                             SMqDataRsp* pDataRsp, int32_t type) {
   uint64_t   consumerId = pRequest->consumerId;
   int32_t    vgId = TD_VID(pTq->pVnode);
@@ -389,7 +392,7 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest,
   return 0;
 }
 
-int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
+int32_t tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
   int32_t        vgId = TD_VID(pTq->pVnode);
   STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen);
 
diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c
new file mode 100644
index 0000000000..50fcea2b54
--- /dev/null
+++ b/source/dnode/vnode/src/tq/tqRestore.c
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tq.h"
+
+static int32_t restoreStreamTask(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList);
+static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
+
+// this function should be executed by stream threads.
+// there is a case that the WAL increases more fast than the restore procedure, and this restore procedure
+// will not stop eventually.
+int tqDoRestoreSourceStreamTasks(STQ* pTq) {
+
+  // todo set the offset value from the previous check point offset
+  int64_t st = taosGetTimestampMs();
+  int32_t vgId = TD_VID(pTq->pVnode);
+  int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pRestoreTasks);
+  tqInfo("vgId:%d start restoring stream tasks, total tasks:%d", vgId, numOfTasks);
+
+  while (1) {
+    SArray* pTaskList = taosArrayInit(4, POINTER_BYTES);
+
+    // check all restore tasks
+    restoreStreamTask(pTq->pStreamMeta, pTq->pOffsetStore, pTaskList);
+    transferToNormalTask(pTq->pStreamMeta, pTaskList);
+    taosArrayDestroy(pTaskList);
+
+    int32_t numOfRestored = taosHashGetSize(pTq->pStreamMeta->pRestoreTasks);
+    if (numOfRestored <= 0) {
+      break;
+    }
+  }
+
+  int64_t et = taosGetTimestampMs();
+  tqInfo("vgId:%d restoring task completed, elapsed time:%" PRId64 " sec.", vgId, (et - st));
+  return 0;
+}
+
+int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
+  int32_t numOfTask = taosArrayGetSize(pTaskList);
+  if (numOfTask <= 0)  {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // todo: add lock
+  for(int32_t i = 0; i < numOfTask; ++i){
+    SStreamTask* pTask = taosArrayGetP(pTaskList, i);
+    tqDebug("vgId:%d transfer s-task:%s state restore -> ready", pStreamMeta->vgId, pTask->id.idStr);
+    taosHashRemove(pStreamMeta->pRestoreTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
+
+    // NOTE: do not change the following order
+    atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
+    taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t restoreStreamTask(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList) {
+  // check all restore tasks
+  void* pIter = NULL;
+
+  while (1) {
+    pIter = taosHashIterate(pStreamMeta->pRestoreTasks, pIter);
+    if (pIter == NULL) {
+      break;
+    }
+
+    SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      continue;
+    }
+
+    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, pTask->taskStatus);
+      continue;
+    }
+
+    // check if offset value exists
+    char key[128] = {0};
+    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
+
+    if (tInputQueueIsFull(pTask)) {
+      tqDebug("s-task:%s input queue is full, do nothing" PRId64, pTask->id.idStr);
+      continue;
+    }
+
+    // check if offset value exists
+    STqOffset* pOffset = tqOffsetRead(pOffsetStore, key);
+    if (pOffset != NULL) {
+      // seek the stored version and extract data from WAL
+      int32_t code = tqSeekVer(pTask->exec.pTqReader, pOffset->val.version, "");
+      if (code == TSDB_CODE_SUCCESS) {  // all data retrieved, abort
+        // append the data for the stream
+        SFetchRet ret = {.data.info.type = STREAM_NORMAL};
+        terrno = 0;
+
+        tqNextBlock(pTask->exec.pTqReader, &ret);
+        if (ret.fetchType == FETCH_TYPE__DATA) {
+          code = launchTaskForWalBlock(pTask, &ret, pOffset);
+          if (code != TSDB_CODE_SUCCESS) {
+            continue;
+          }
+        } else {
+          // FETCH_TYPE__NONE: all data has been retrieved from WAL, let's try submit block directly.
+          tqDebug("s-task:%s data in WAL are all consumed, transfer this task to be normal state", pTask->id.idStr);
+          taosArrayPush(pTaskList, &pTask);
+        }
+      } else {  // failed to seek to the WAL version
+        tqDebug("s-task:%s data in WAL are all consumed, transfer this task to be normal state", pTask->id.idStr);
+        taosArrayPush(pTaskList, &pTask);
+      }
+    } else {
+      ASSERT(0);
+    }
+  }
+
+  return 0;
+}
+
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
new file mode 100644
index 0000000000..ac88cf1916
--- /dev/null
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tq.h"
+
+// stream_task:stream_id:task_id
+void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
+  int32_t n = 12;
+  char* p = dst;
+
+  memcpy(p, "stream_task:", n);
+  p += n;
+
+  int32_t inc = tintToHex(streamId, p);
+  p += inc;
+
+  *(p++) = ':';
+  tintToHex(taskId, p);
+}
+
+int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
+  int32_t code = tAppendDataForStream(pTask, pQueueItem);
+  if (code < 0) {
+    tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver);
+    return -1;
+  }
+
+  if (streamSchedExec(pTask) < 0) {
+    tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno));
+    return -1;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset) {
+  SStreamDataBlock* pBlocks = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
+  if (pBlocks == NULL) { // failed, do nothing
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pRet->data.info.type = STREAM_NORMAL;
+  pBlocks->type = STREAM_INPUT__DATA_BLOCK;
+  pBlocks->sourceVer = pOffset->val.version;
+  pBlocks->blocks = taosArrayInit(0, sizeof(SSDataBlock));
+  taosArrayPush(pBlocks->blocks, &pRet->data);
+
+//          int64_t* ts = (int64_t*)(((SColumnInfoData*)ret.data.pDataBlock->pData)->pData);
+//          tqDebug("-----------%ld\n", ts[0]);
+
+  int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pBlocks, pBlocks->sourceVer);
+  if (code == TSDB_CODE_SUCCESS) {
+    pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pTqReader->pWalReader);
+    tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
+            pOffset->val.version);
+  }
+
+  return 0;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index d681f5b65e..eb3c5d1f64 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -549,6 +549,9 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
 
   pVnode->restored = true;
   vInfo("vgId:%d, sync restore finished", pVnode->config.vgId);
+
+  // start to restore all stream tasks
+  tqRestoreStreamTasks(pVnode->pTq);
 }
 
 static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 577f6d6e00..8693915c46 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -45,11 +45,17 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     goto _err;
   }
 
-  pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
+  pMeta->pTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK);
   if (pMeta->pTasks == NULL) {
     goto _err;
   }
 
+  pMeta->pRestoreTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK);
+  if (pMeta->pRestoreTasks == NULL) {
+    goto _err;
+  }
+
   if (streamMetaBegin(pMeta) < 0) {
     goto _err;
   }
@@ -62,6 +68,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
 _err:
   taosMemoryFree(pMeta->path);
   if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks);
+  if (pMeta->pRestoreTasks) taosHashCleanup(pMeta->pRestoreTasks);
   if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb);
   if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb);
   if (pMeta->db) tdbClose(pMeta->db);
@@ -87,8 +94,9 @@ void streamMetaClose(SStreamMeta* pMeta) {
     tFreeStreamTask(pTask);
     /*streamMetaReleaseTask(pMeta, pTask);*/
   }
+
   taosHashCleanup(pMeta->pTasks);
-  taosHashCleanup(pMeta->pRecoverStatus);
+  taosHashCleanup(pMeta->pRestoreTasks);
   taosMemoryFree(pMeta->path);
   taosMemoryFree(pMeta);
 }
@@ -166,8 +174,7 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
     return -1;
   }
 
-  taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*));
-
+  taosHashPut(pMeta->pRestoreTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES);
   return 0;
 }
 #endif
@@ -298,12 +305,13 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
       return -1;
     }
 
-    if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
+    if (taosHashPut(pMeta->pRestoreTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
       tdbFree(pKey);
       tdbFree(pVal);
       tdbTbcClose(pCur);
       return -1;
     }
+
     /*pTask->taskStatus = TASK_STATUS__NORMAL;*/
     if (pTask->fillHistory) {
       pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index ad6127ead2..e20299be38 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -100,6 +100,8 @@ int32_t walNextValidMsg(SWalReader *pReader) {
   return -1;
 }
 
+int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; }
+
 static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) {
   int64_t ret = 0;
 

From a4ba5401f96b505c6edf6cb4f70c7af57a2bb76d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 11 Apr 2023 10:01:36 +0800
Subject: [PATCH 03/65] enh(stream): set the start version of all operators.

---
 include/libs/executor/executor.h              |  3 +
 include/libs/stream/tstream.h                 | 33 ++++-----
 source/dnode/vnode/src/inc/tq.h               |  4 +
 source/dnode/vnode/src/tq/tq.c                | 57 +++-----------
 source/dnode/vnode/src/tq/tqCommit.c          |  3 +
 source/dnode/vnode/src/tq/tqRestore.c         | 17 ++---
 source/dnode/vnode/src/tq/tqUtil.c            | 74 ++++++++++++++++++-
 source/libs/executor/inc/executorimpl.h       | 16 ++--
 source/libs/executor/src/executor.c           |  6 ++
 source/libs/executor/src/timewindowoperator.c | 15 ++++
 source/libs/stream/src/streamExec.c           | 22 +++++-
 source/libs/stream/src/streamMeta.c           | 46 +++++++++---
 source/util/src/tworker.c                     |  2 +-
 13 files changed, 200 insertions(+), 98 deletions(-)

diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index ee8ee1050d..fd66194143 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -91,6 +91,9 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
 
 int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
 
+// todo refactor
+int64_t qGetCheckpointVersion(qTaskInfo_t tinfo);
+
 /**
  * Set multiple input data blocks for the stream scan.
  * @param tinfo
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index a338413502..22e9356b3d 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -295,14 +295,11 @@ struct SStreamTask {
   int16_t   dispatchMsgType;
   int8_t    taskStatus;
   int8_t    schedStatus;
-
-  // node info
-  int32_t selfChildId;
-  int32_t nodeId;
-  SEpSet  epSet;
-
-  int64_t recoverSnapVer;
-  int64_t startVer;
+  int32_t   selfChildId;
+  int32_t   nodeId;
+  SEpSet    epSet;
+  int64_t   recoverSnapVer;
+  int64_t   startVer;
 
   // fill history
   int8_t fillHistory;
@@ -340,15 +337,15 @@ struct SStreamTask {
   // state backend
   SStreamState* pState;
 
-  // do not serialize
-  int32_t recoverTryingDownstream;
-  int32_t recoverWaitingUpstream;
-  int64_t checkReqId;
-  SArray* checkReqIds;  // shuffle
-  int32_t refCnt;
-
-  int64_t    checkpointingId;
-  int32_t    checkpointAlignCnt;
+  // the followings attributes don't be serialized
+  int32_t             recoverTryingDownstream;
+  int32_t             recoverWaitingUpstream;
+  int64_t             checkReqId;
+  SArray*             checkReqIds;  // shuffle
+  int32_t             refCnt;
+  int64_t             checkpointingId;
+  int32_t             checkpointAlignCnt;
+  struct SStreamMeta* pMeta;
 };
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -597,6 +594,8 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 void         streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
 
+SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId);
+
 int32_t streamMetaBegin(SStreamMeta* pMeta);
 int32_t streamMetaCommit(SStreamMeta* pMeta);
 int32_t streamMetaRollBack(SStreamMeta* pMeta);
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index d4af9ac481..50a09229fa 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -183,6 +183,10 @@ void    createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId)
 int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver);
 int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset);
 
+void    doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver);
+void    saveOffsetForAllTasks(STQ* pTq, int64_t ver);
+void    initOffsetForAllRestoreTasks(STQ* pTq);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index aa8960e977..91e2569a8c 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -907,6 +907,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
   pTask->pMsgCb = &pTq->pVnode->msgCb;
   pTask->startVer = ver;
+  pTask->pMeta = pTq->pStreamMeta;
 
   // expand executor
   if (pTask->fillHistory) {
@@ -979,7 +980,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   }
 
   streamSetupTrigger(pTask);
-  tqInfo("vgId:%d expand stream task, s-task:%s, child id %d, level %d", vgId, pTask->id.idStr, pTask->selfChildId, pTask->taskLevel);
+  tqInfo("vgId:%d expand stream task, s-task:%s, ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr,
+         pTask->startVer, pTask->selfChildId, pTask->taskLevel);
   return 0;
 }
 
@@ -1370,16 +1372,6 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   return 0;
 }
 
-static void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
-  STqOffset offset = {0};
-  tqOffsetResetToLog(&offset.val, ver);
-
-  tstrncpy(offset.subKey, pKey, tListLen(offset.subKey));
-
-  // keep the offset info in the offset store
-  tqOffsetWrite(pOffsetStore, &offset);
-}
-
 static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit,
                                          const char* key, int64_t ver) {
   doSaveTaskOffset(pOffsetStore, key, ver);
@@ -1392,36 +1384,6 @@ static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTa
   return TSDB_CODE_SUCCESS;
 }
 
-static void saveOffsetForAllTasks(STQ* pTq, SPackedData submit) {
-  void* pIter = NULL;
-
-  while(1) {
-    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) {
-      break;
-    }
-
-    SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
-      continue;
-    }
-
-    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
-              pTask->taskStatus);
-      continue;
-    }
-
-    char key[128] = {0};
-    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
-
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
-    if (pOffset == NULL) {
-      doSaveTaskOffset(pTq->pOffsetStore, key, submit.ver);
-    }
-  }
-}
-
 int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
   void* pIter = NULL;
 
@@ -1429,7 +1391,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
   if (pSubmit == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tqError("failed to create data submit for stream since out of memory");
-    saveOffsetForAllTasks(pTq, submit);
+    saveOffsetForAllTasks(pTq, submit.ver);
     return -1;
   }
 
@@ -1518,11 +1480,14 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
     tqDoRestoreSourceStreamTasks(pTq);
     return 0;
   } else {
-    SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
+    SStreamTask* pTask = streamMetaAcquireTaskEx(pTq->pStreamMeta, taskId);
     if (pTask != NULL) {
       if (pTask->taskStatus == TASK_STATUS__NORMAL) {
         tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
         streamProcessRunReq(pTask);
+      } else if (pTask->taskStatus == TASK_STATUS_RESTORE) {
+        tqDebug("vgId:%d s-task:%s start to restore from last ck", vgId, pTask->id.idStr);
+        streamProcessRunReq(pTask);
       } else {
         tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr);
       }
@@ -1683,8 +1648,10 @@ int32_t tqRestoreStreamTasks(STQ* pTq) {
     return -1;
   }
 
-  tqInfo("vgId:%d start to restore all stream tasks", vgId);
-  
+  int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pRestoreTasks);
+  tqInfo("vgId:%d start restoring stream tasks, total tasks:%d", vgId, numOfTasks);
+  initOffsetForAllRestoreTasks(pTq);
+
   pRunReq->head.vgId = vgId;
   pRunReq->streamId = 0;
   pRunReq->taskId = ALL_STREAM_TASKS_ID;
diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c
index 7fc66c4919..0f5daa31ad 100644
--- a/source/dnode/vnode/src/tq/tqCommit.c
+++ b/source/dnode/vnode/src/tq/tqCommit.c
@@ -16,10 +16,13 @@
 #include "tq.h"
 
 int tqCommit(STQ* pTq) {
+#if 0
+  // stream meta commit does not be aligned to the vnode commit
   if (streamMetaCommit(pTq->pStreamMeta) < 0) {
     tqError("vgId:%d, failed to commit stream meta since %s", TD_VID(pTq->pVnode), terrstr());
     return -1;
   }
+#endif
 
   return tqOffsetCommitFile(pTq->pOffsetStore);
 }
diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c
index 50fcea2b54..9377e3d58f 100644
--- a/source/dnode/vnode/src/tq/tqRestore.c
+++ b/source/dnode/vnode/src/tq/tqRestore.c
@@ -15,25 +15,19 @@
 
 #include "tq.h"
 
-static int32_t restoreStreamTask(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList);
+static int32_t restoreStreamTaskImpl(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList);
 static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
 
 // this function should be executed by stream threads.
 // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure
 // will not stop eventually.
 int tqDoRestoreSourceStreamTasks(STQ* pTq) {
-
-  // todo set the offset value from the previous check point offset
   int64_t st = taosGetTimestampMs();
-  int32_t vgId = TD_VID(pTq->pVnode);
-  int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pRestoreTasks);
-  tqInfo("vgId:%d start restoring stream tasks, total tasks:%d", vgId, numOfTasks);
-
   while (1) {
     SArray* pTaskList = taosArrayInit(4, POINTER_BYTES);
 
     // check all restore tasks
-    restoreStreamTask(pTq->pStreamMeta, pTq->pOffsetStore, pTaskList);
+    restoreStreamTaskImpl(pTq->pStreamMeta, pTq->pOffsetStore, pTaskList);
     transferToNormalTask(pTq->pStreamMeta, pTaskList);
     taosArrayDestroy(pTaskList);
 
@@ -44,7 +38,7 @@ int tqDoRestoreSourceStreamTasks(STQ* pTq) {
   }
 
   int64_t et = taosGetTimestampMs();
-  tqInfo("vgId:%d restoring task completed, elapsed time:%" PRId64 " sec.", vgId, (et - st));
+  tqInfo("vgId:%d restoring task completed, elapsed time:%" PRId64 " sec.", TD_VID(pTq->pVnode), (et - st));
   return 0;
 }
 
@@ -68,7 +62,7 @@ int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t restoreStreamTask(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList) {
+int32_t restoreStreamTaskImpl(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList) {
   // check all restore tasks
   void* pIter = NULL;
 
@@ -93,7 +87,8 @@ int32_t restoreStreamTask(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore
     createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
 
     if (tInputQueueIsFull(pTask)) {
-      tqDebug("s-task:%s input queue is full, do nothing" PRId64, pTask->id.idStr);
+      tqDebug("s-task:%s input queue is full, do nothing", pTask->id.idStr);
+      taosMsleep(10);
       continue;
     }
 
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index ac88cf1916..14054ad998 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -69,4 +69,76 @@ int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pO
   }
 
   return 0;
-}
\ No newline at end of file
+}
+
+void initOffsetForAllRestoreTasks(STQ* pTq) {
+  void* pIter = NULL;
+
+  while(1) {
+    pIter = taosHashIterate(pTq->pStreamMeta->pRestoreTasks, pIter);
+    if (pIter == NULL) {
+      break;
+    }
+
+    SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      continue;
+    }
+
+    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
+              pTask->taskStatus);
+      continue;
+    }
+
+    char key[128] = {0};
+    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
+
+    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
+    if (pOffset == NULL) {
+      doSaveTaskOffset(pTq->pOffsetStore, key, pTask->startVer);
+    }
+  }
+
+}
+
+void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
+  void* pIter = NULL;
+
+  while(1) {
+    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
+    if (pIter == NULL) {
+      break;
+    }
+
+    SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      continue;
+    }
+
+    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
+              pTask->taskStatus);
+      continue;
+    }
+
+    char key[128] = {0};
+    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
+
+    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
+    if (pOffset == NULL) {
+      doSaveTaskOffset(pTq->pOffsetStore, key, ver);
+    }
+  }
+}
+
+void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
+  STqOffset offset = {0};
+  tqOffsetResetToLog(&offset.val, ver);
+
+  tstrncpy(offset.subKey, pKey, tListLen(offset.subKey));
+
+  // keep the offset info in the offset store
+  tqOffsetWrite(pOffsetStore, &offset);
+}
+
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 16433dc34e..759502e40f 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -127,14 +127,9 @@ enum {
 };
 
 typedef struct {
-  // TODO remove prepareStatus
-//  STqOffsetVal prepareStatus;  // for tmq
-  STqOffsetVal currentOffset;     // for tmq
-  SMqMetaRsp   metaRsp;        // for tmq fetching meta
-//  int8_t       returned;
-  int64_t      snapshotVer;
-  // const SSubmitReq* pReq;
-
+  STqOffsetVal        currentOffset;  // for tmq
+  SMqMetaRsp          metaRsp;        // for tmq fetching meta
+  int64_t             snapshotVer;
   SPackedData         submit;
   SSchemaWrapper*     schema;
   char                tbName[TSDB_TABLE_NAME_LEN];
@@ -144,6 +139,7 @@ typedef struct {
   int64_t             fillHistoryVer1;
   int64_t             fillHistoryVer2;
   SStreamState*       pState;
+  int64_t             dataVersion;
 } SStreamTaskInfo;
 
 typedef struct {
@@ -191,7 +187,6 @@ enum {
   OP_OPENED = 0x1,
   OP_RES_TO_RETURN = 0x5,
   OP_EXEC_DONE = 0x9,
-//  OP_EXEC_RECV = 0x11,
 };
 
 typedef struct SOperatorFpSet {
@@ -560,6 +555,7 @@ typedef struct SStreamIntervalOperatorInfo {
   uint64_t           numOfDatapack;
   SArray*            pUpdated;
   SSHashObj*         pUpdatedMap;
+  int64_t            dataVersion;
 } SStreamIntervalOperatorInfo;
 
 typedef struct SDataGroupInfo {
@@ -609,6 +605,7 @@ typedef struct SStreamSessionAggOperatorInfo {
   bool                ignoreExpiredDataSaved;
   SArray*             pUpdated;
   SSHashObj*          pStUpdated;
+  int64_t             dataVersion;
 } SStreamSessionAggOperatorInfo;
 
 typedef struct SStreamStateAggOperatorInfo {
@@ -627,6 +624,7 @@ typedef struct SStreamStateAggOperatorInfo {
   bool                ignoreExpiredDataSaved;
   SArray*             pUpdated;
   SSHashObj*          pSeUpdated;
+  int64_t             dataVersion;
 } SStreamStateAggOperatorInfo;
 
 typedef struct SStreamPartitionOperatorInfo {
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index d9868f59b9..caaeaa76c2 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -198,6 +198,12 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
   return code;
 }
 
+int64_t qGetCheckpointVersion(qTaskInfo_t tinfo) {
+  SExecTaskInfo* pTaskInfo = tinfo;
+  return pTaskInfo->streamInfo.dataVersion;
+}
+
+
 int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) {
   if (tinfo == NULL) {
     return TSDB_CODE_APP_ERROR;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 880de7d6bf..f122323109 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -2333,9 +2333,14 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN
   return startPos;
 }
 
+static void setStreamDataVersion(SExecTaskInfo*  pTaskInfo, int64_t version) {
+  pTaskInfo->streamInfo.dataVersion = version;
+}
+
 static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId,
                                     SSHashObj* pUpdatedMap) {
   SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;
+  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
 
   SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
   SExecTaskInfo*  pTaskInfo = pOperatorInfo->pTaskInfo;
@@ -2501,6 +2506,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       clearFunctionContext(&pOperator->exprSupp);
       // semi interval operator clear disk buffer
       clearStreamIntervalOperator(pInfo);
+      setStreamDataVersion(pTaskInfo, pInfo->dataVersion);
       qDebug("===stream===clear semi operator");
     } else {
       deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark,
@@ -2774,6 +2780,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   pInfo->numOfDatapack = 0;
   pInfo->pUpdated = NULL;
   pInfo->pUpdatedMap = NULL;
+  pInfo->dataVersion = 0;
 
   pOperator->operatorType = pPhyNode->type;
   pOperator->blocking = true;
@@ -3124,6 +3131,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
   int32_t                        rows = pSDataBlock->info.rows;
   int32_t                        winRows = 0;
 
+  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
+
   SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
   TSKEY*           startTsCols = (int64_t*)pStartTsCol->pData;
   SColumnInfoData* pEndTsCol = NULL;
@@ -3587,6 +3596,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->ignoreExpiredDataSaved = false;
   pInfo->pUpdated = NULL;
   pInfo->pStUpdated = NULL;
+  pInfo->dataVersion = 0;
 
   setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
                   OP_NOT_OPENED, pInfo, pTaskInfo);
@@ -3897,6 +3907,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   TSKEY*                       tsCols = NULL;
   SResultRow*                  pResult = NULL;
   int32_t                      winRows = 0;
+
+  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
+
   if (pSDataBlock->pDataBlock != NULL) {
     SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
     tsCols = (int64_t*)pColDataInfo->pData;
@@ -4113,6 +4126,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->ignoreExpiredDataSaved = false;
   pInfo->pUpdated = NULL;
   pInfo->pSeUpdated = NULL;
+  pInfo->dataVersion = 0;
 
   setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
                   pInfo, pTaskInfo);
@@ -4748,6 +4762,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
                           &pInfo->delKey);
     setOperatorCompleted(pOperator);
     streamStateCommit(pTaskInfo->streamInfo.pState);
+    setStreamDataVersion(pTaskInfo, pInfo->dataVersion);
     return NULL;
   }
 
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index d23590a08b..f2db8113d3 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -21,8 +21,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
   int32_t code = TSDB_CODE_SUCCESS;
   void*   pExecutor = pTask->exec.pExecutor;
 
-  while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
-    qError("stream task wait for the end of fill history");
+  while (pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
+    qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr,
+           atomic_load_8(&pTask->taskStatus));
     taosMsleep(2);
     continue;
   }
@@ -236,7 +237,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
     while (1) {
       SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
       if (qItem == NULL) {
-        qDebug("stream task exec over, queue empty, task: %d", pTask->id.taskId);
+        qDebug("s-task:%s stream task exec over, queue empty", pTask->id.idStr);
         break;
       }
 
@@ -284,7 +285,19 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 
     streamTaskExecImpl(pTask, pInput, pRes);
 
-    qDebug("s-task:%s exec end", pTask->id.idStr);
+    int64_t ckVer = qGetCheckpointVersion(pTask->exec.pExecutor);
+    if (ckVer > pTask->startVer) {    // save it since the checkpoint is updated
+      qDebug("s-task:%s exec end, checkpoint ver from %"PRId64" to %"PRId64, pTask->id.idStr, pTask->startVer, ckVer);
+      pTask->startVer = ckVer;
+      streamMetaSaveTask(pTask->pMeta, pTask);
+
+      if (streamMetaCommit(pTask->pMeta) < 0) {
+        qError("failed to commit stream meta, since %s", terrstr());
+        return -1;
+      }
+    } else {
+      qDebug("s-task:%s exec end", pTask->id.idStr);
+    }
 
     if (taosArrayGetSize(pRes) != 0) {
       SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
@@ -333,6 +346,7 @@ int32_t streamTryExec(SStreamTask* pTask) {
       return -1;
     }
 
+    // todo the task should be commit here
     atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE);
 
     if (!taosQueueEmpty(pTask->inputQueue->queue)) {
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 8693915c46..a22d768a89 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -195,17 +195,12 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
   taosRLockLatch(&pMeta->lock);
 
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
-  if (ppTask) {
-    SStreamTask* pTask = *ppTask;
-    if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__DROPPING) {
-      atomic_add_fetch_32(&pTask->refCnt, 1);
-      taosRUnLockLatch(&pMeta->lock);
-      return pTask;
-    } else {
-      taosRUnLockLatch(&pMeta->lock);
-      return NULL;
-    }
+  if (ppTask != NULL && (atomic_load_8(&((*ppTask)->taskStatus)) != TASK_STATUS__DROPPING)) {
+    atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
+    taosRUnLockLatch(&pMeta->lock);
+    return *ppTask;
   }
+
   taosRUnLockLatch(&pMeta->lock);
   return NULL;
 }
@@ -219,6 +214,37 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   }
 }
 
+SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId) {
+  taosRLockLatch(&pMeta->lock);
+
+  SStreamTask* pTask = NULL;
+  int32_t numOfRestored = taosHashGetSize(pMeta->pRestoreTasks);
+  if (numOfRestored > 0) {
+    SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pRestoreTasks, &taskId, sizeof(int32_t));
+    if (p != NULL) {
+      pTask = *p;
+      if (pTask != NULL && (atomic_load_8(&(pTask->taskStatus)) != TASK_STATUS__DROPPING)) {
+        atomic_add_fetch_32(&pTask->refCnt, 1);
+        taosRUnLockLatch(&pMeta->lock);
+        return pTask;
+      }
+    }
+  } else {
+    SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
+    if (p != NULL) {
+      pTask = *p;
+      if (pTask != NULL && atomic_load_8(&(pTask->taskStatus)) != TASK_STATUS__DROPPING) {
+        atomic_add_fetch_32(&pTask->refCnt, 1);
+        taosRUnLockLatch(&pMeta->lock);
+        return pTask;
+      }
+    }
+  }
+
+  taosRUnLockLatch(&pMeta->lock);
+  return NULL;
+}
+
 void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
   if (ppTask) {
diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c
index 631bcb443e..d57104dd78 100644
--- a/source/util/src/tworker.c
+++ b/source/util/src/tworker.c
@@ -218,7 +218,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem
   int32_t queueNum = taosGetQueueNumber(pool->qset);
   int32_t curWorkerNum = taosArrayGetSize(pool->workers);
   int32_t dstWorkerNum = ceil(queueNum * pool->ratio);
-  if (dstWorkerNum < 1) dstWorkerNum = 1;
+  if (dstWorkerNum < 2) dstWorkerNum = 2;
 
   // spawn a thread to process queue
   while (curWorkerNum < dstWorkerNum) {

From 74da3c05bcad0ceef999f09c1dc63719c76ed985 Mon Sep 17 00:00:00 2001
From: liuyao <54liuyao@163.com>
Date: Tue, 11 Apr 2023 10:33:10 +0800
Subject: [PATCH 04/65] feat:set check point id

---
 include/libs/executor/executor.h              |  2 +-
 include/libs/stream/streamState.h             |  1 +
 source/libs/executor/inc/executorimpl.h       |  1 +
 source/libs/executor/src/executor.c           |  5 +++--
 source/libs/executor/src/timewindowoperator.c |  7 ++++---
 source/libs/stream/src/streamExec.c           | 10 ++++++----
 source/libs/stream/src/streamState.c          |  2 ++
 7 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index fd66194143..9d05d1fdc9 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -92,7 +92,7 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
 int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
 
 // todo refactor
-int64_t qGetCheckpointVersion(qTaskInfo_t tinfo);
+void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId);
 
 /**
  * Set multiple input data blocks for the stream scan.
diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h
index fd5cec2931..42a7261f38 100644
--- a/include/libs/stream/streamState.h
+++ b/include/libs/stream/streamState.h
@@ -42,6 +42,7 @@ typedef struct STdbState {
 typedef struct {
   STdbState* pTdbState;
   int32_t    number;
+  int64_t    checkPointId;
 } SStreamState;
 
 SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 759502e40f..7fee4f9b83 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -140,6 +140,7 @@ typedef struct {
   int64_t             fillHistoryVer2;
   SStreamState*       pState;
   int64_t             dataVersion;
+  int64_t             checkPointId;
 } SStreamTaskInfo;
 
 typedef struct {
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index caaeaa76c2..2244847faa 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -198,9 +198,10 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
   return code;
 }
 
-int64_t qGetCheckpointVersion(qTaskInfo_t tinfo) {
+void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId) {
   SExecTaskInfo* pTaskInfo = tinfo;
-  return pTaskInfo->streamInfo.dataVersion;
+  *dataVer = pTaskInfo->streamInfo.dataVersion;
+  *ckId = pTaskInfo->streamInfo.checkPointId;
 }
 
 
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index f122323109..658dd51978 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -2333,8 +2333,9 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN
   return startPos;
 }
 
-static void setStreamDataVersion(SExecTaskInfo*  pTaskInfo, int64_t version) {
+static void setStreamDataVersion(SExecTaskInfo*  pTaskInfo, int64_t version, int64_t ckId) {
   pTaskInfo->streamInfo.dataVersion = version;
+  pTaskInfo->streamInfo.checkPointId = ckId;
 }
 
 static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId,
@@ -2506,7 +2507,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       clearFunctionContext(&pOperator->exprSupp);
       // semi interval operator clear disk buffer
       clearStreamIntervalOperator(pInfo);
-      setStreamDataVersion(pTaskInfo, pInfo->dataVersion);
+      setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId);
       qDebug("===stream===clear semi operator");
     } else {
       deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark,
@@ -4762,7 +4763,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
                           &pInfo->delKey);
     setOperatorCompleted(pOperator);
     streamStateCommit(pTaskInfo->streamInfo.pState);
-    setStreamDataVersion(pTaskInfo, pInfo->dataVersion);
+    setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId);
     return NULL;
   }
 
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index f2db8113d3..8dafafcc5f 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -285,10 +285,12 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 
     streamTaskExecImpl(pTask, pInput, pRes);
 
-    int64_t ckVer = qGetCheckpointVersion(pTask->exec.pExecutor);
-    if (ckVer > pTask->startVer) {    // save it since the checkpoint is updated
-      qDebug("s-task:%s exec end, checkpoint ver from %"PRId64" to %"PRId64, pTask->id.idStr, pTask->startVer, ckVer);
-      pTask->startVer = ckVer;
+    int64_t ckId = 0;
+    int64_t dataVer = 0;
+    qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId);
+    if (dataVer > pTask->startVer) {    // save it since the checkpoint is updated
+      qDebug("s-task:%s exec end, checkpoint ver from %"PRId64" to %"PRId64, pTask->id.idStr, pTask->startVer, dataVer);
+      pTask->startVer = dataVer;
       streamMetaSaveTask(pTask->pMeta, pTask);
 
       if (streamMetaCommit(pTask->pMeta) < 0) {
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 04a1414438..7bea989e3a 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -193,6 +193,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
   }
 
   pState->pTdbState->pOwner = pTask;
+  pState->checkPointId = 0;
 
   return pState;
 
@@ -243,6 +244,7 @@ int32_t streamStateCommit(SStreamState* pState) {
                TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
     return -1;
   }
+  pState->checkPointId++;
   return 0;
 }
 

From 15cceb5a5aa234bdb124287670abb74852af8923 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 11 Apr 2023 11:53:10 +0800
Subject: [PATCH 05/65] refactor: do some internal refactors.

---
 include/libs/stream/tstream.h          | 34 ++++++++++++++------------
 source/dnode/snode/src/snode.c         |  2 +-
 source/dnode/vnode/src/tq/tq.c         |  6 ++---
 source/dnode/vnode/src/tq/tqUtil.c     |  2 +-
 source/dnode/vnode/src/tsdb/tsdbRead.c | 16 ++++++------
 source/libs/stream/src/streamExec.c    |  6 ++---
 source/libs/stream/src/streamTask.c    |  8 +++---
 7 files changed, 38 insertions(+), 36 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 22e9356b3d..7dcb2e1796 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -287,19 +287,24 @@ typedef struct SStreamId {
   const char* idStr;
 } SStreamId;
 
+typedef struct SCheckpointInfo {
+  int64_t id;
+  int64_t version;   // offset in WAL
+} SCheckpointInfo;
+
 struct SStreamTask {
-  SStreamId id;
-  int32_t   totalLevel;
-  int8_t    taskLevel;
-  int8_t    outputType;
-  int16_t   dispatchMsgType;
-  int8_t    taskStatus;
-  int8_t    schedStatus;
-  int32_t   selfChildId;
-  int32_t   nodeId;
-  SEpSet    epSet;
-  int64_t   recoverSnapVer;
-  int64_t   startVer;
+  SStreamId       id;
+  int32_t         totalLevel;
+  int8_t          taskLevel;
+  int8_t          outputType;
+  int16_t         dispatchMsgType;
+  int8_t          taskStatus;
+  int8_t          schedStatus;
+  int32_t         selfChildId;
+  int32_t         nodeId;
+  SEpSet          epSet;
+  SCheckpointInfo chkInfo;
+  STaskExec       exec;
 
   // fill history
   int8_t fillHistory;
@@ -309,9 +314,6 @@ struct SStreamTask {
   int32_t nextCheckId;
   SArray* checkpointInfo;  // SArray<SStreamCheckpointInfo>
 
-  // exec
-  STaskExec exec;
-
   // output
   union {
     STaskDispatcherFixedEp fixedEpDispatcher;
@@ -587,7 +589,7 @@ void         streamMetaClose(SStreamMeta* streamMeta);
 
 int32_t      streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
-int32_t      streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen);
+int32_t      streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen);
 // SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
 
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 447c90eb58..ee6f649d52 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -76,7 +76,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
   pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
   pTask->pMsgCb = &pSnode->msgCb;
-  pTask->startVer = ver;
+  pTask->chkInfo.version = ver;
 
   pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1);
   if (pTask->pState == NULL) {
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 91e2569a8c..afdbf4e7c8 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -906,7 +906,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
   pTask->pMsgCb = &pTq->pVnode->msgCb;
-  pTask->startVer = ver;
+  pTask->chkInfo.version = ver;
   pTask->pMeta = pTq->pStreamMeta;
 
   // expand executor
@@ -981,7 +981,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
 
   streamSetupTrigger(pTask);
   tqInfo("vgId:%d expand stream task, s-task:%s, ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr,
-         pTask->startVer, pTask->selfChildId, pTask->taskLevel);
+         pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel);
   return 0;
 }
 
@@ -1124,7 +1124,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   // check param
-  int64_t fillVer1 = pTask->startVer;
+  int64_t fillVer1 = pTask->chkInfo.version;
   if (fillVer1 <= 0) {
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     return -1;
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 14054ad998..5f4e0ced11 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -96,7 +96,7 @@ void initOffsetForAllRestoreTasks(STQ* pTq) {
 
     STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
     if (pOffset == NULL) {
-      doSaveTaskOffset(pTq->pOffsetStore, key, pTask->startVer);
+      doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version);
     }
   }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index b80c952ee0..61f7747ae4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2088,7 +2088,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
       pBlockScanInfo->lastKey = tsLastBlock;
       return TSDB_CODE_SUCCESS;
     } else {
-      int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema);
+      code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
@@ -2112,7 +2112,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
       }
     }
   } else {  // not merge block data
-    int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema);
+    code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -2352,7 +2352,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
         tsdbRowMergerAdd(&merge, piRow, piSchema);
       } else {
         init = true;
-        STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
+        pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
         code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema);
         if (code != TSDB_CODE_SUCCESS) {
           return code;
@@ -2575,7 +2575,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
     SRow*      pTSRow = NULL;
     SRowMerger merge = {0};
 
-    int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema);
+    code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -3242,8 +3242,8 @@ static int32_t readRowsCountFromFiles(STsdbReader* pReader) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   while (1) {
-    bool    hasNext = false;
-    int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext);
+    bool hasNext = false;
+    code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext);
     if (code) {
       return code;
     }
@@ -3515,8 +3515,8 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_
   int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion;
 
   int64_t endVer = 0;
-  if (pCond->endVersion ==
-      -1) {  // user not specified end version, set current maximum version of vnode as the endVersion
+  if (pCond->endVersion == -1) {
+    // user not specified end version, set current maximum version of vnode as the endVersion
     endVer = pVnode->state.applied;
   } else {
     endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion;
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 8dafafcc5f..98052ec6ba 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -288,9 +288,9 @@ int32_t streamExecForAll(SStreamTask* pTask) {
     int64_t ckId = 0;
     int64_t dataVer = 0;
     qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId);
-    if (dataVer > pTask->startVer) {    // save it since the checkpoint is updated
-      qDebug("s-task:%s exec end, checkpoint ver from %"PRId64" to %"PRId64, pTask->id.idStr, pTask->startVer, dataVer);
-      pTask->startVer = dataVer;
+    if (dataVer > pTask->chkInfo.version) {    // save it since the checkpoint is updated
+      qDebug("s-task:%s exec end, checkpoint ver from %"PRId64" to %"PRId64, pTask->id.idStr, pTask->chkInfo.version, dataVer);
+      pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId};
       streamMetaSaveTask(pTask->pMeta, pTask);
 
       if (streamMetaCommit(pTask->pMeta) < 0) {
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index c1f50178dd..b1f0a63c2e 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -70,8 +70,8 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
   if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1;
 
-  if (tEncodeI64(pEncoder, pTask->recoverSnapVer) < 0) return -1;
-  if (tEncodeI64(pEncoder, pTask->startVer) < 0) return -1;
+  if (tEncodeI64(pEncoder, pTask->chkInfo.id) < 0) return -1;
+  if (tEncodeI64(pEncoder, pTask->chkInfo.version) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->fillHistory) < 0) return -1;
 
   int32_t epSz = taosArrayGetSize(pTask->childEpInfo);
@@ -123,8 +123,8 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;
   if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1;
 
-  if (tDecodeI64(pDecoder, &pTask->recoverSnapVer) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pTask->startVer) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pTask->chkInfo.id) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pTask->chkInfo.version) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->fillHistory) < 0) return -1;
 
   int32_t epSz;

From 18479d8115f26c52e726b56d75ddb76bcada6209 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 11 Apr 2023 15:42:24 +0800
Subject: [PATCH 06/65] refactor: do some internall refactor.

---
 include/libs/stream/tstream.h              |  8 ++++----
 source/dnode/mnode/impl/src/mndDef.c       |  4 ++--
 source/dnode/mnode/impl/src/mndScheduler.c | 10 +++++-----
 source/dnode/mnode/impl/src/mndStream.c    |  8 ++++----
 source/dnode/snode/src/snode.c             |  2 +-
 source/dnode/vnode/src/tq/tq.c             | 10 +++++-----
 source/dnode/vnode/src/tq/tqRestore.c      |  5 +++--
 source/libs/stream/src/streamExec.c        | 14 +++++++++-----
 source/libs/stream/src/streamMeta.c        |  8 ++++----
 source/libs/stream/src/streamTask.c        |  6 +++---
 10 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 7dcb2e1796..2368788824 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -50,7 +50,7 @@ enum {
   TASK_STATUS__RECOVER_PREPARE,
   TASK_STATUS__RECOVER1,
   TASK_STATUS__RECOVER2,
-  TASK_STATUS_RESTORE,  // only available for source task to replay WAL from the checkpoint
+  TASK_STATUS__RESTORE,  // only available for source task to replay WAL from the checkpoint
 };
 
 enum {
@@ -353,9 +353,9 @@ struct SStreamTask {
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
 int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
 
-SStreamTask* tNewSStreamTask(int64_t streamId);
-int32_t      tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
-int32_t      tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
+SStreamTask* tNewStreamTask(int64_t streamId);
+int32_t      tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
+int32_t      tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
 void         tFreeStreamTask(SStreamTask* pTask);
 int32_t      tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem);
 bool         tInputQueueIsFull(const SStreamTask* pTask);
diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index e221a64619..c69f08eb6b 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -70,7 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
     if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
     for (int32_t j = 0; j < innerSz; j++) {
       SStreamTask *pTask = taosArrayGetP(pArray, j);
-      if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1;
+      if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1;
     }
   }
 
@@ -130,7 +130,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
           taosArrayDestroy(pArray);
           return -1;
         }
-        if (tDecodeSStreamTask(pDecoder, pTask) < 0) {
+        if (tDecodeStreamTask(pDecoder, pTask) < 0) {
           taosMemoryFree(pTask);
           taosArrayDestroy(pArray);
           return -1;
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 504749df49..36521fd778 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -224,7 +224,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
       continue;
     }
 
-    SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+    SStreamTask* pTask = tNewStreamTask(pStream->uid);
     if (pTask == NULL) {
       sdbRelease(pSdb, pVgroup);
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -260,7 +260,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
 
 int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) {
   SArray*      tasks = taosArrayGetP(pStream->tasks, 0);
-  SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+  SStreamTask* pTask = tNewStreamTask(pStream->uid);
   if (pTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
@@ -350,7 +350,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
         return -1;
       }
 
-      pInnerTask = tNewSStreamTask(pStream->uid);
+      pInnerTask = tNewStreamTask(pStream->uid);
       if (pInnerTask == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         qDestroyQueryPlan(pPlan);
@@ -421,7 +421,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
         continue;
       }
 
-      SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+      SStreamTask* pTask = tNewStreamTask(pStream->uid);
       if (pTask == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         sdbRelease(pSdb, pVgroup);
@@ -491,7 +491,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
         continue;
       }
 
-      SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+      SStreamTask* pTask = tNewStreamTask(pStream->uid);
       if (pTask == NULL) {
         sdbRelease(pSdb, pVgroup);
         qDestroyQueryPlan(pPlan);
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index a0118ee749..b8c540266f 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -39,8 +39,8 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
 static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
-// static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
-/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/
+static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
+static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq);
 static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
 static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
@@ -418,7 +418,7 @@ FAIL:
 int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
   SEncoder encoder;
   tEncoderInit(&encoder, NULL, 0);
-  tEncodeSStreamTask(&encoder, pTask);
+  tEncodeStreamTask(&encoder, pTask);
   int32_t size = encoder.pos;
   int32_t tlen = sizeof(SMsgHead) + size;
   tEncoderClear(&encoder);
@@ -430,7 +430,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
   ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId);
   void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
   tEncoderInit(&encoder, abuf, size);
-  tEncodeSStreamTask(&encoder, pTask);
+  tEncodeStreamTask(&encoder, pTask);
   tEncoderClear(&encoder);
 
   STransAction action = {0};
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index ee6f649d52..f8e4268aad 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -139,7 +139,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
   }
   SDecoder decoder;
   tDecoderInit(&decoder, (uint8_t *)msg, msgLen);
-  code = tDecodeSStreamTask(&decoder, pTask);
+  code = tDecodeStreamTask(&decoder, pTask);
   if (code < 0) {
     tDecoderClear(&decoder);
     taosMemoryFree(pTask);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index afdbf4e7c8..6fca9cc808 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -906,14 +906,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
   pTask->pMsgCb = &pTq->pVnode->msgCb;
-  pTask->chkInfo.version = ver;
   pTask->pMeta = pTq->pStreamMeta;
 
   // expand executor
   if (pTask->fillHistory) {
     pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
   } else {
-    pTask->taskStatus = TASK_STATUS_RESTORE;
+    pTask->taskStatus = TASK_STATUS__RESTORE;
   }
 
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
@@ -1089,7 +1088,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
 
   SDecoder decoder;
   tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
-  code = tDecodeSStreamTask(&decoder, pTask);
+  code = tDecodeStreamTask(&decoder, pTask);
   if (code < 0) {
     tDecoderClear(&decoder);
     taosMemoryFree(pTask);
@@ -1485,8 +1484,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
       if (pTask->taskStatus == TASK_STATUS__NORMAL) {
         tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
         streamProcessRunReq(pTask);
-      } else if (pTask->taskStatus == TASK_STATUS_RESTORE) {
-        tqDebug("vgId:%d s-task:%s start to restore from last ck", vgId, pTask->id.idStr);
+      } else if (pTask->taskStatus == TASK_STATUS__RESTORE) {
+        tqDebug("vgId:%d s-task:%s start to process in restore procedure from last chk point:%" PRId64, vgId,
+                pTask->id.idStr, pTask->chkInfo.version);
         streamProcessRunReq(pTask);
       } else {
         tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr);
diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c
index 9377e3d58f..a123bdb1dc 100644
--- a/source/dnode/vnode/src/tq/tqRestore.c
+++ b/source/dnode/vnode/src/tq/tqRestore.c
@@ -49,9 +49,10 @@ int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
   }
 
   // todo: add lock
-  for(int32_t i = 0; i < numOfTask; ++i){
+  for (int32_t i = 0; i < numOfTask; ++i) {
     SStreamTask* pTask = taosArrayGetP(pTaskList, i);
-    tqDebug("vgId:%d transfer s-task:%s state restore -> ready", pStreamMeta->vgId, pTask->id.idStr);
+    tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
+            pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
     taosHashRemove(pStreamMeta->pRestoreTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
 
     // NOTE: do not change the following order
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 98052ec6ba..075e477eb3 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -21,11 +21,15 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
   int32_t code = TSDB_CODE_SUCCESS;
   void*   pExecutor = pTask->exec.pExecutor;
 
-  while (pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
-    qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr,
-           atomic_load_8(&pTask->taskStatus));
-    taosMsleep(2);
-    continue;
+  while (pTask->taskLevel == TASK_LEVEL__SOURCE) {
+    int8_t status = atomic_load_8(&pTask->taskStatus);
+    if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__RESTORE) {
+      qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr,
+             atomic_load_8(&pTask->taskStatus));
+      taosMsleep(2);
+    } else {
+      break;
+    }
   }
 
   // set input
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index a22d768a89..ae65753bed 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -109,7 +109,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg,
   }
   SDecoder decoder;
   tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
-  if (tDecodeSStreamTask(&decoder, pTask) < 0) {
+  if (tDecodeStreamTask(&decoder, pTask) < 0) {
     tDecoderClear(&decoder);
     goto FAIL;
   }
@@ -142,7 +142,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   void*   buf = NULL;
   int32_t len;
   int32_t code;
-  tEncodeSize(tEncodeSStreamTask, pTask, len, code);
+  tEncodeSize(tEncodeStreamTask, pTask, len, code);
   if (code < 0) {
     return -1;
   }
@@ -153,7 +153,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
 
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, len);
-  tEncodeSStreamTask(&encoder, pTask);
+  tEncodeStreamTask(&encoder, pTask);
   tEncoderClear(&encoder);
 
   if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) {
@@ -321,7 +321,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
       return -1;
     }
     tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
-    tDecodeSStreamTask(&decoder, pTask);
+    tDecodeStreamTask(&decoder, pTask);
     tDecoderClear(&decoder);
 
     if (pMeta->expandFunc(pMeta->ahandle, pTask, -1) < 0) {
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index b1f0a63c2e..f45b6ad7b7 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -16,7 +16,7 @@
 #include "executor.h"
 #include "tstream.h"
 
-SStreamTask* tNewSStreamTask(int64_t streamId) {
+SStreamTask* tNewStreamTask(int64_t streamId) {
   SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
   if (pTask == NULL) {
     return NULL;
@@ -54,7 +54,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
   return 0;
 }
 
-int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
+int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   if (tStartEncode(pEncoder) < 0) return -1;
   if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1;
@@ -107,7 +107,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   return pEncoder->pos;
 }
 
-int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
+int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   if (tStartDecode(pDecoder) < 0) return -1;
   if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1;

From aae275886352a704787940933b94b226dda4693c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 11 Apr 2023 16:58:51 +0800
Subject: [PATCH 07/65] fix(stream): disable the deploy msg when restart taosd.

---
 source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index e353988d4c..76ff04b81a 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -400,7 +400,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       }
       break;
     case TDMT_STREAM_TASK_DEPLOY: {
-      if (tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) {
+      if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) {
         goto _err;
       }
     } break;

From f083697dba5091531c03874deefb395adcfe2e8e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 11 Apr 2023 19:04:33 +0800
Subject: [PATCH 08/65] refactor(tq): do some internal refactor.

---
 source/dnode/vnode/src/inc/tq.h    |   1 +
 source/dnode/vnode/src/tq/tq.c     | 346 +----------------------------
 source/dnode/vnode/src/tq/tqUtil.c | 346 +++++++++++++++++++++++++++++
 3 files changed, 348 insertions(+), 345 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index 50a09229fa..94ba399a0a 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -182,6 +182,7 @@ int32_t tqDoRestoreSourceStreamTasks(STQ* pTq);
 void    createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId);
 int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver);
 int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset);
+int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
 
 void    doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver);
 void    saveOffsetForAllTasks(STQ* pTq, int64_t ver);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 6fca9cc808..48c9a4f445 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -15,7 +15,6 @@
 
 #include "tq.h"
 
-#define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)
 #define ALL_STREAM_TASKS_ID       (-1)
 
 int32_t tqInit() {
@@ -143,44 +142,6 @@ void tqClose(STQ* pTq) {
   taosMemoryFree(pTq);
 }
 
-int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) {
-  int32_t len = 0;
-  int32_t code = 0;
-  tEncodeSize(tEncodeSMqMetaRsp, pRsp, len, code);
-  if (code < 0) {
-    return -1;
-  }
-  int32_t tlen = sizeof(SMqRspHead) + len;
-  void*   buf = rpcMallocCont(tlen);
-  if (buf == NULL) {
-    return -1;
-  }
-
-  ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP;
-  ((SMqRspHead*)buf)->epoch = pReq->epoch;
-  ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
-
-  void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, abuf, len);
-  tEncodeSMqMetaRsp(&encoder, pRsp);
-  tEncoderClear(&encoder);
-
-  SRpcMsg resp = {
-      .info = pMsg->info,
-      .pCont = buf,
-      .contLen = tlen,
-      .code = 0,
-  };
-  tmsgSendRsp(&resp);
-
-  tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d",
-          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type);
-
-  return 0;
-}
-
 static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch,
                              int64_t consumerId, int32_t type) {
   int32_t len = 0;
@@ -330,311 +291,6 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) {
   return 0;
 }
 
-static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
-  pRsp->reqOffset = pReq->reqOffset;
-
-  pRsp->blockData = taosArrayInit(0, sizeof(void*));
-  pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
-
-  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) {
-    return -1;
-  }
-
-  pRsp->withTbName = 0;
-  pRsp->withSchema = false;
-  return 0;
-}
-
-static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
-  pRsp->reqOffset = pReq->reqOffset;
-
-  pRsp->withTbName = 1;
-  pRsp->withSchema = 1;
-  pRsp->blockData = taosArrayInit(0, sizeof(void*));
-  pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
-  pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
-  pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
-
-  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
-                                     SRpcMsg* pMsg, bool* pBlockReturned) {
-  uint64_t     consumerId = pRequest->consumerId;
-  STqOffsetVal reqOffset = pRequest->reqOffset;
-  STqOffset*   pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey);
-  int32_t      vgId = TD_VID(pTq->pVnode);
-
-  *pBlockReturned = false;
-
-  // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value.
-  if (pOffset != NULL) {
-    *pOffsetVal = pOffset->val;
-
-    char formatBuf[80];
-    tFormatOffset(formatBuf, 80, pOffsetVal);
-    tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%"PRIx64,
-            consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId);
-    return 0;
-  } else {
-    // no poll occurs in this vnode for this topic, let's seek to the right offset value.
-    if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
-      if (pRequest->useSnapshot) {
-        tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot",
-                consumerId, pHandle->subKey, vgId);
-
-        if (pHandle->fetchMeta) {
-          tqOffsetResetToMeta(pOffsetVal, 0);
-        } else {
-          tqOffsetResetToData(pOffsetVal, 0, 0);
-        }
-      } else {
-        pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef);
-        if (pHandle->pRef == NULL) {
-          terrno = TSDB_CODE_OUT_OF_MEMORY;
-          return -1;
-        }
-
-        // offset set to previous version when init
-        tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1);
-      }
-    } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
-      if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-        SMqDataRsp dataRsp = {0};
-        tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
-
-        tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
-        tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
-                pHandle->subKey, vgId, dataRsp.rspOffset.version);
-        int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
-        tDeleteSMqDataRsp(&dataRsp);
-
-        *pBlockReturned = true;
-        return code;
-      } else {
-        STaosxRsp taosxRsp = {0};
-        tqInitTaosxRsp(&taosxRsp, pRequest);
-        tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
-        int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
-        tDeleteSTaosxRsp(&taosxRsp);
-
-        *pBlockReturned = true;
-        return code;
-      }
-    } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
-      tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed",
-              pHandle->subKey, consumerId, vgId, pRequest->subKey);
-      terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
-                                                   SRpcMsg* pMsg, STqOffsetVal* pOffset) {
-  uint64_t consumerId = pRequest->consumerId;
-  int32_t  vgId = TD_VID(pTq->pVnode);
-
-  SMqDataRsp dataRsp = {0};
-  tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
-
-  // lock
-  taosWLockLatch(&pTq->lock);
-
-  qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
-  int code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
-  if(code != 0) {
-    goto end;
-  }
-
-  // till now, all data has been transferred to consumer, new data needs to push client once arrived.
-  if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
-      dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
-    code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
-    taosWUnLockLatch(&pTq->lock);
-    return code;
-  }
-
-
-  code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
-
-  // NOTE: this pHandle->consumerId may have been changed already.
-
-end:
-  {
-    char buf[80] = {0};
-    tFormatOffset(buf, 80, &dataRsp.rspOffset);
-    tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d",
-            consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code);
-    taosWUnLockLatch(&pTq->lock);
-    tDeleteSMqDataRsp(&dataRsp);
-  }
-  return code;
-}
-
-
-static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) {
-  int code = 0;
-  int32_t      vgId = TD_VID(pTq->pVnode);
-  SWalCkHead*  pCkHead = NULL;
-  SMqMetaRsp metaRsp = {0};
-  STaosxRsp taosxRsp = {0};
-  tqInitTaosxRsp(&taosxRsp, pRequest);
-
-  if (offset->type != TMQ_OFFSET__LOG) {
-    if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) {
-      return -1;
-    }
-
-    if (metaRsp.metaRspLen > 0) {
-      code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp);
-      tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64,
-              pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts);
-      taosMemoryFree(metaRsp.metaRsp);
-      tDeleteSTaosxRsp(&taosxRsp);
-      return code;
-    }
-
-    tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64
-    ",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts);
-    if (taosxRsp.blockNum > 0) {
-      code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
-      tDeleteSTaosxRsp(&taosxRsp);
-      return code;
-    }else {
-      *offset = taosxRsp.rspOffset;
-    }
-  }
-
-
-  if (offset->type == TMQ_OFFSET__LOG) {
-    int64_t fetchVer = offset->version + 1;
-    pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
-    if (pCkHead == NULL) {
-      tDeleteSTaosxRsp(&taosxRsp);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-    walSetReaderCapacity(pHandle->pWalReader, 2048);
-    int totalRows = 0;
-    while (1) {
-      int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
-      if (savedEpoch > pRequest->epoch) {
-        tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64
-          ", found new consumer epoch %d, discard req epoch %d", pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch);
-        break;
-      }
-
-      if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) {
-        tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
-        code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
-        tDeleteSTaosxRsp(&taosxRsp);
-        taosMemoryFreeClear(pCkHead);
-        return code;
-      }
-
-      SWalCont* pHead = &pCkHead->head;
-      tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId,
-              pRequest->epoch, vgId, fetchVer, pHead->msgType);
-
-      // process meta
-      if (pHead->msgType != TDMT_VND_SUBMIT) {
-        if(totalRows > 0) {
-          tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1);
-          code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
-          tDeleteSTaosxRsp(&taosxRsp);
-          taosMemoryFreeClear(pCkHead);
-          return code;
-        }
-
-        tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
-        tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer);
-        metaRsp.resMsgType = pHead->msgType;
-        metaRsp.metaRspLen = pHead->bodyLen;
-        metaRsp.metaRsp = pHead->body;
-        if (tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp) < 0) {
-          code = -1;
-          taosMemoryFreeClear(pCkHead);
-          tDeleteSTaosxRsp(&taosxRsp);
-          return code;
-        }
-        code = 0;
-        taosMemoryFreeClear(pCkHead);
-        tDeleteSTaosxRsp(&taosxRsp);
-        return code;
-      }
-
-      // process data
-      SPackedData submit = {
-          .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)),
-          .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg),
-          .ver = pHead->version,
-      };
-
-      if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) {
-        tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId,
-                pRequest->subKey);
-        taosMemoryFreeClear(pCkHead);
-        tDeleteSTaosxRsp(&taosxRsp);
-        return -1;
-      }
-
-      if (totalRows >= 4096 || taosxRsp.createTableNum > 0) {
-        tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
-        code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
-        tDeleteSTaosxRsp(&taosxRsp);
-        taosMemoryFreeClear(pCkHead);
-        return code;
-      } else {
-        fetchVer++;
-      }
-    }
-  }
-
-  tDeleteSTaosxRsp(&taosxRsp);
-  taosMemoryFreeClear(pCkHead);
-  return 0;
-}
-
-static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
-  int32_t      code = -1;
-  STqOffsetVal offset = {0};
-  STqOffsetVal reqOffset = pRequest->reqOffset;
-
-  // 1. reset the offset if needed
-  if (IS_OFFSET_RESET_TYPE(reqOffset.type)) {
-    // handle the reset offset cases, according to the consumer's choice.
-    bool blockReturned = false;
-    code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned);
-    if (code != 0) {
-      return code;
-    }
-
-    // empty block returned, quit
-    if (blockReturned) {
-      return 0;
-    }
-  } else { // use the consumer specified offset
-    // the offset value can not be monotonious increase??
-    offset = reqOffset;
-  }
-
-  // this is a normal subscribe requirement
-  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
-  }
-
-  // todo handle the case where re-balance occurs.
-  // for taosx
-  return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
-}
-
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   SMqPollReq   req = {0};
   if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
@@ -681,7 +337,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
           consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
 
-  return doPollDataForMq(pTq, pHandle, &req, pMsg);
+  return tqExtractDataForMq(pTq, pHandle, &req, pMsg);
 }
 
 int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 5f4e0ced11..8b86cb6716 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -15,6 +15,10 @@
 
 #include "tq.h"
 
+#define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)
+
+static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp);
+
 // stream_task:stream_id:task_id
 void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
   int32_t n = 12;
@@ -142,3 +146,345 @@ void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ve
   tqOffsetWrite(pOffsetStore, &offset);
 }
 
+static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
+  pRsp->reqOffset = pReq->reqOffset;
+
+  pRsp->blockData = taosArrayInit(0, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
+
+  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) {
+    return -1;
+  }
+
+  pRsp->withTbName = 0;
+  pRsp->withSchema = false;
+  return 0;
+}
+
+static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
+  pRsp->reqOffset = pReq->reqOffset;
+
+  pRsp->withTbName = 1;
+  pRsp->withSchema = 1;
+  pRsp->blockData = taosArrayInit(0, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
+  pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
+  pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
+
+  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
+                                     SRpcMsg* pMsg, bool* pBlockReturned) {
+  uint64_t     consumerId = pRequest->consumerId;
+  STqOffsetVal reqOffset = pRequest->reqOffset;
+  STqOffset*   pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey);
+  int32_t      vgId = TD_VID(pTq->pVnode);
+
+  *pBlockReturned = false;
+
+  // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value.
+  if (pOffset != NULL) {
+    *pOffsetVal = pOffset->val;
+
+    char formatBuf[80];
+    tFormatOffset(formatBuf, 80, pOffsetVal);
+    tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%"PRIx64,
+            consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId);
+    return 0;
+  } else {
+    // no poll occurs in this vnode for this topic, let's seek to the right offset value.
+    if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
+      if (pRequest->useSnapshot) {
+        tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot",
+                consumerId, pHandle->subKey, vgId);
+
+        if (pHandle->fetchMeta) {
+          tqOffsetResetToMeta(pOffsetVal, 0);
+        } else {
+          tqOffsetResetToData(pOffsetVal, 0, 0);
+        }
+      } else {
+        pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef);
+        if (pHandle->pRef == NULL) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          return -1;
+        }
+
+        // offset set to previous version when init
+        tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1);
+      }
+    } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
+      if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+        SMqDataRsp dataRsp = {0};
+        tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
+
+        tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
+        tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
+                pHandle->subKey, vgId, dataRsp.rspOffset.version);
+        int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
+        tDeleteSMqDataRsp(&dataRsp);
+
+        *pBlockReturned = true;
+        return code;
+      } else {
+        STaosxRsp taosxRsp = {0};
+        tqInitTaosxRsp(&taosxRsp, pRequest);
+        tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
+        int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
+        tDeleteSTaosxRsp(&taosxRsp);
+
+        *pBlockReturned = true;
+        return code;
+      }
+    } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
+      tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed",
+              pHandle->subKey, consumerId, vgId, pRequest->subKey);
+      terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
+                                                   SRpcMsg* pMsg, STqOffsetVal* pOffset) {
+  uint64_t consumerId = pRequest->consumerId;
+  int32_t  vgId = TD_VID(pTq->pVnode);
+
+  SMqDataRsp dataRsp = {0};
+  tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
+
+  // lock
+  taosWLockLatch(&pTq->lock);
+
+  qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
+  int code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
+  if(code != 0) {
+    goto end;
+  }
+
+  // till now, all data has been transferred to consumer, new data needs to push client once arrived.
+  if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
+      dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
+    code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
+    taosWUnLockLatch(&pTq->lock);
+    return code;
+  }
+
+
+  code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
+
+  // NOTE: this pHandle->consumerId may have been changed already.
+
+  end:
+  {
+    char buf[80] = {0};
+    tFormatOffset(buf, 80, &dataRsp.rspOffset);
+    tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d",
+            consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code);
+    taosWUnLockLatch(&pTq->lock);
+    tDeleteSMqDataRsp(&dataRsp);
+  }
+  return code;
+}
+
+
+static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) {
+  int code = 0;
+  int32_t      vgId = TD_VID(pTq->pVnode);
+  SWalCkHead*  pCkHead = NULL;
+  SMqMetaRsp metaRsp = {0};
+  STaosxRsp taosxRsp = {0};
+  tqInitTaosxRsp(&taosxRsp, pRequest);
+
+  if (offset->type != TMQ_OFFSET__LOG) {
+    if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) {
+      return -1;
+    }
+
+    if (metaRsp.metaRspLen > 0) {
+      code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp);
+      tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64,
+              pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts);
+      taosMemoryFree(metaRsp.metaRsp);
+      tDeleteSTaosxRsp(&taosxRsp);
+      return code;
+    }
+
+    tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64
+                ",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts);
+    if (taosxRsp.blockNum > 0) {
+      code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
+      tDeleteSTaosxRsp(&taosxRsp);
+      return code;
+    }else {
+      *offset = taosxRsp.rspOffset;
+    }
+  }
+
+
+  if (offset->type == TMQ_OFFSET__LOG) {
+    int64_t fetchVer = offset->version + 1;
+    pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
+    if (pCkHead == NULL) {
+      tDeleteSTaosxRsp(&taosxRsp);
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    walSetReaderCapacity(pHandle->pWalReader, 2048);
+    int totalRows = 0;
+    while (1) {
+      int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
+      if (savedEpoch > pRequest->epoch) {
+        tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64
+                   ", found new consumer epoch %d, discard req epoch %d", pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch);
+        break;
+      }
+
+      if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) {
+        tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
+        code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
+        tDeleteSTaosxRsp(&taosxRsp);
+        taosMemoryFreeClear(pCkHead);
+        return code;
+      }
+
+      SWalCont* pHead = &pCkHead->head;
+      tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId,
+              pRequest->epoch, vgId, fetchVer, pHead->msgType);
+
+      // process meta
+      if (pHead->msgType != TDMT_VND_SUBMIT) {
+        if(totalRows > 0) {
+          tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1);
+          code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
+          tDeleteSTaosxRsp(&taosxRsp);
+          taosMemoryFreeClear(pCkHead);
+          return code;
+        }
+
+        tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
+        tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer);
+        metaRsp.resMsgType = pHead->msgType;
+        metaRsp.metaRspLen = pHead->bodyLen;
+        metaRsp.metaRsp = pHead->body;
+        if (tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp) < 0) {
+          code = -1;
+          taosMemoryFreeClear(pCkHead);
+          tDeleteSTaosxRsp(&taosxRsp);
+          return code;
+        }
+        code = 0;
+        taosMemoryFreeClear(pCkHead);
+        tDeleteSTaosxRsp(&taosxRsp);
+        return code;
+      }
+
+      // process data
+      SPackedData submit = {
+          .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)),
+          .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg),
+          .ver = pHead->version,
+      };
+
+      if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) {
+        tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId,
+                pRequest->subKey);
+        taosMemoryFreeClear(pCkHead);
+        tDeleteSTaosxRsp(&taosxRsp);
+        return -1;
+      }
+
+      if (totalRows >= 4096 || taosxRsp.createTableNum > 0) {
+        tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
+        code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
+        tDeleteSTaosxRsp(&taosxRsp);
+        taosMemoryFreeClear(pCkHead);
+        return code;
+      } else {
+        fetchVer++;
+      }
+    }
+  }
+
+  tDeleteSTaosxRsp(&taosxRsp);
+  taosMemoryFreeClear(pCkHead);
+  return 0;
+}
+
+int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
+  int32_t      code = -1;
+  STqOffsetVal offset = {0};
+  STqOffsetVal reqOffset = pRequest->reqOffset;
+
+  // 1. reset the offset if needed
+  if (IS_OFFSET_RESET_TYPE(reqOffset.type)) {
+    // handle the reset offset cases, according to the consumer's choice.
+    bool blockReturned = false;
+    code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned);
+    if (code != 0) {
+      return code;
+    }
+
+    // empty block returned, quit
+    if (blockReturned) {
+      return 0;
+    }
+  } else { // use the consumer specified offset
+    // the offset value can not be monotonious increase??
+    offset = reqOffset;
+  }
+
+  // this is a normal subscribe requirement
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+    return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
+  }
+
+  // todo handle the case where re-balance occurs.
+  // for taosx
+  return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
+}
+
+int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) {
+  int32_t len = 0;
+  int32_t code = 0;
+  tEncodeSize(tEncodeSMqMetaRsp, pRsp, len, code);
+  if (code < 0) {
+    return -1;
+  }
+  int32_t tlen = sizeof(SMqRspHead) + len;
+  void*   buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    return -1;
+  }
+
+  ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP;
+  ((SMqRspHead*)buf)->epoch = pReq->epoch;
+  ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
+
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, abuf, len);
+  tEncodeSMqMetaRsp(&encoder, pRsp);
+  tEncoderClear(&encoder);
+
+  SRpcMsg resp = {
+      .info = pMsg->info,
+      .pCont = buf,
+      .contLen = tlen,
+      .code = 0,
+  };
+  tmsgSendRsp(&resp);
+
+  tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d",
+          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type);
+
+  return 0;
+}

From 5ab54481f0d8b974f60d549a8bce9124a3c8223b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 11 Apr 2023 19:24:34 +0800
Subject: [PATCH 09/65] refactor: do some internal refactor.

---
 include/libs/stream/tstream.h          | 96 ++++++++++----------------
 source/dnode/snode/src/snode.c         |  2 +-
 source/dnode/vnode/src/tq/tq.c         | 22 +++---
 source/dnode/vnode/src/tq/tqRestore.c  |  6 +-
 source/dnode/vnode/src/tq/tqUtil.c     |  8 +--
 source/libs/stream/src/stream.c        |  9 +--
 source/libs/stream/src/streamExec.c    | 16 ++---
 source/libs/stream/src/streamMeta.c    | 26 ++-----
 source/libs/stream/src/streamRecover.c | 10 +--
 source/libs/stream/src/streamTask.c    | 10 +--
 10 files changed, 86 insertions(+), 119 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 2368788824..9f7d366a46 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "executor.h"
 #include "os.h"
+#include "executor.h"
 #include "query.h"
 #include "streamState.h"
 #include "tdatablock.h"
@@ -104,21 +104,8 @@ typedef struct {
   int8_t type;
 } SStreamQueueItem;
 
-#if 0
-typedef struct {
-  int8_t      type;
-  int64_t     ver;
-  int32_t*    dataRef;
-  SSubmitReq* data;
-} SStreamDataSubmit;
-
-typedef struct {
-  int8_t  type;
-  int64_t ver;
-  SArray* dataRefs;  // SArray<int32_t*>
-  SArray* reqs;      // SArray<SSubmitReq*>
-} SStreamMergedSubmit;
-#endif
+typedef void    FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
+typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver);
 
 typedef struct {
   int8_t      type;
@@ -220,7 +207,6 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) {
 }
 
 static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) {
-  //
   return queue->qItem;
 }
 
@@ -249,16 +235,13 @@ typedef struct {
   SUseDbRsp dbInfo;
 } STaskDispatcherShuffle;
 
-typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
-
 typedef struct {
   int64_t         stbUid;
   char            stbFullName[TSDB_TABLE_FNAME_LEN];
   SSchemaWrapper* pSchemaWrapper;
-  // not applicable to encoder and decoder
-  void*     vnode;
-  FTbSink*  tbSinkFunc;
-  STSchema* pTSchema;
+  void*           vnode;  // not available to encoder and decoder
+  FTbSink*        tbSinkFunc;
+  STSchema*       pTSchema;
 } STaskSinkTb;
 
 typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data);
@@ -292,14 +275,18 @@ typedef struct SCheckpointInfo {
   int64_t version;   // offset in WAL
 } SCheckpointInfo;
 
+typedef struct SStreamStatus {
+  int8_t taskStatus;
+  int8_t schedStatus;
+} SStreamStatus;
+
 struct SStreamTask {
   SStreamId       id;
   int32_t         totalLevel;
   int8_t          taskLevel;
   int8_t          outputType;
   int16_t         dispatchMsgType;
-  int8_t          taskStatus;
-  int8_t          schedStatus;
+  SStreamStatus   status;
   int32_t         selfChildId;
   int32_t         nodeId;
   SEpSet          epSet;
@@ -329,15 +316,11 @@ struct SStreamTask {
   SStreamQueue* outputQueue;
 
   // trigger
-  int8_t  triggerStatus;
-  int64_t triggerParam;
-  void*   timer;
-
-  // msg handle
-  SMsgCb* pMsgCb;
-
-  // state backend
-  SStreamState* pState;
+  int8_t        triggerStatus;
+  int64_t       triggerParam;
+  void*         timer;
+  SMsgCb*       pMsgCb;  // msg handle
+  SStreamState* pState;  // state backend
 
   // the followings attributes don't be serialized
   int32_t             recoverTryingDownstream;
@@ -350,6 +333,21 @@ struct SStreamTask {
   struct SStreamMeta* pMeta;
 };
 
+// meta
+typedef struct SStreamMeta {
+    char*        path;
+    TDB*         db;
+    TTB*         pTaskDb;
+    TTB*         pCheckpointDb;
+    SHashObj*    pTasks;
+    SHashObj*    pRestoreTasks;
+    void*        ahandle;
+    TXN*         txn;
+    FTaskExpand* expandFunc;
+    int32_t      vgId;
+    SRWLatch     lock;
+} SStreamMeta;
+
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
 int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
 
@@ -566,42 +564,22 @@ int32_t streamAggRecoverPrepare(SStreamTask* pTask);
 // int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask);
 int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId);
 
-// expand and deploy
-typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver);
-
-// meta
-typedef struct SStreamMeta {
-  char*        path;
-  TDB*         db;
-  TTB*         pTaskDb;
-  TTB*         pCheckpointDb;
-  SHashObj*    pTasks;
-  SHashObj*    pRestoreTasks;
-  void*        ahandle;
-  TXN*         txn;
-  FTaskExpand* expandFunc;
-  int32_t      vgId;
-  SRWLatch     lock;
-} SStreamMeta;
-
 SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId);
 void         streamMetaClose(SStreamMeta* streamMeta);
 
 int32_t      streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
 int32_t      streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen);
-// SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
 
+SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 void         streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
 
-SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId);
-
-int32_t streamMetaBegin(SStreamMeta* pMeta);
-int32_t streamMetaCommit(SStreamMeta* pMeta);
-int32_t streamMetaRollBack(SStreamMeta* pMeta);
-int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver);
+int32_t      streamMetaBegin(SStreamMeta* pMeta);
+int32_t      streamMetaCommit(SStreamMeta* pMeta);
+int32_t      streamMetaRollBack(SStreamMeta* pMeta);
+int32_t      streamLoadTasks(SStreamMeta* pMeta, int64_t ver);
 
 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index f8e4268aad..7ccbb3b586 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -65,7 +65,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
   ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
 
   pTask->refCnt = 1;
-  pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
+  pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
   pTask->inputQueue = streamQueueOpen();
   pTask->outputQueue = streamQueueOpen();
 
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 48c9a4f445..b029afc935 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -551,7 +551,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   int32_t vgId = TD_VID(pTq->pVnode);
   pTask->id.idStr = taosStrdup(buf);
   pTask->refCnt = 1;
-  pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
+  pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
   pTask->inputQueue = streamQueueOpen();
   pTask->outputQueue = streamQueueOpen();
 
@@ -566,9 +566,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
 
   // expand executor
   if (pTask->fillHistory) {
-    pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
+    pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
   } else {
-    pTask->taskStatus = TASK_STATUS__RESTORE;
+    pTask->status.taskStatus = TASK_STATUS__RESTORE;
   }
 
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
@@ -661,7 +661,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
   };
 
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
-  if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) {
+  if (pTask && atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL) {
     rsp.status = 1;
   } else {
     rsp.status = 0;
@@ -788,7 +788,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
   // do recovery step 1
   streamSourceRecoverScanStep1(pTask);
 
-  if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
+  if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     return 0;
   }
@@ -803,7 +803,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
 
   streamMetaReleaseTask(pTq->pStreamMeta, pTask);
 
-  if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
+  if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
     return 0;
   }
 
@@ -845,7 +845,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t
     return -1;
   }
 
-  if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
+  if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     return 0;
   }
@@ -1061,9 +1061,9 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
       continue;
     }
 
-    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
       tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
-              pTask->taskStatus);
+              pTask->status.taskStatus);
       continue;
     }
 
@@ -1137,10 +1137,10 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   } else {
     SStreamTask* pTask = streamMetaAcquireTaskEx(pTq->pStreamMeta, taskId);
     if (pTask != NULL) {
-      if (pTask->taskStatus == TASK_STATUS__NORMAL) {
+      if (pTask->status.taskStatus == TASK_STATUS__NORMAL) {
         tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
         streamProcessRunReq(pTask);
-      } else if (pTask->taskStatus == TASK_STATUS__RESTORE) {
+      } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) {
         tqDebug("vgId:%d s-task:%s start to process in restore procedure from last chk point:%" PRId64, vgId,
                 pTask->id.idStr, pTask->chkInfo.version);
         streamProcessRunReq(pTask);
diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c
index a123bdb1dc..877c686d35 100644
--- a/source/dnode/vnode/src/tq/tqRestore.c
+++ b/source/dnode/vnode/src/tq/tqRestore.c
@@ -56,7 +56,7 @@ int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
     taosHashRemove(pStreamMeta->pRestoreTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
 
     // NOTE: do not change the following order
-    atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
+    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
     taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
   }
 
@@ -78,8 +78,8 @@ int32_t restoreStreamTaskImpl(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS
       continue;
     }
 
-    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, pTask->taskStatus);
+    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, pTask->status.taskStatus);
       continue;
     }
 
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 8b86cb6716..2e8c6a53bb 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -89,9 +89,9 @@ void initOffsetForAllRestoreTasks(STQ* pTq) {
       continue;
     }
 
-    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
       tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
-              pTask->taskStatus);
+              pTask->status.taskStatus);
       continue;
     }
 
@@ -120,9 +120,9 @@ void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
       continue;
     }
 
-    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
       tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
-              pTask->taskStatus);
+              pTask->status.taskStatus);
       continue;
     }
 
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 59ac8a61d6..71d4e5efd8 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -52,7 +52,7 @@ void streamCleanUp() {
 void streamSchedByTimer(void* param, void* tmrId) {
   SStreamTask* pTask = (void*)param;
 
-  if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
+  if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
     streamMetaReleaseTask(NULL, pTask);
     return;
   }
@@ -66,8 +66,8 @@ void streamSchedByTimer(void* param, void* tmrId) {
       taosFreeQitem(trigger);
       return;
     }
-    trigger->pBlock->info.type = STREAM_GET_ALL;
 
+    trigger->pBlock->info.type = STREAM_GET_ALL;
     atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE);
 
     if (tAppendDataForStream(pTask, (SStreamQueueItem*)trigger) < 0) {
@@ -75,6 +75,7 @@ void streamSchedByTimer(void* param, void* tmrId) {
       taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer);
       return;
     }
+
     streamSchedExec(pTask);
   }
 
@@ -93,13 +94,13 @@ int32_t streamSetupTrigger(SStreamTask* pTask) {
 
 int32_t streamSchedExec(SStreamTask* pTask) {
   int8_t schedStatus =
-      atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING);
+      atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING);
 
   if (schedStatus == TASK_SCHED_STATUS__INACTIVE) {
     SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
     if (pRunReq == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE);
+      atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
       return -1;
     }
 
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 075e477eb3..db9be593c0 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -22,10 +22,10 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
   void*   pExecutor = pTask->exec.pExecutor;
 
   while (pTask->taskLevel == TASK_LEVEL__SOURCE) {
-    int8_t status = atomic_load_8(&pTask->taskStatus);
+    int8_t status = atomic_load_8(&pTask->status.taskStatus);
     if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__RESTORE) {
       qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr,
-             atomic_load_8(&pTask->taskStatus));
+             atomic_load_8(&pTask->status.taskStatus));
       taosMsleep(2);
     } else {
       break;
@@ -66,7 +66,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
 
   // pExecutor
   while (1) {
-    if (pTask->taskStatus == TASK_STATUS__DROPPING) {
+    if (pTask->status.taskStatus == TASK_STATUS__DROPPING) {
       return 0;
     }
 
@@ -134,7 +134,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
 
     int32_t batchCnt = 0;
     while (1) {
-      if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
+      if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
         taosArrayDestroy(pRes);
         return 0;
       }
@@ -267,7 +267,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
       }
     }
 
-    if (pTask->taskStatus == TASK_STATUS__DROPPING) {
+    if (pTask->status.taskStatus == TASK_STATUS__DROPPING) {
       if (pInput) {
         streamFreeQitem(pInput);
       }
@@ -343,17 +343,17 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 int32_t streamTryExec(SStreamTask* pTask) {
   // this function may be executed by multi-threads, so status check is required.
   int8_t schedStatus =
-      atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE);
+      atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE);
 
   if (schedStatus == TASK_SCHED_STATUS__WAITING) {
     int32_t code = streamExecForAll(pTask);
     if (code < 0) {
-      atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__FAILED);
+      atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED);
       return -1;
     }
 
     // todo the task should be commit here
-    atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE);
+    atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
 
     if (!taosQueueEmpty(pTask->inputQueue->queue)) {
       streamSchedExec(pTask);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index ae65753bed..2e9bb4d762 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -179,23 +179,11 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
 }
 #endif
 
-#if 0
-SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) {
-  SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
-  if (ppTask) {
-    ASSERT((*ppTask)->taskId == taskId);
-    return *ppTask;
-  } else {
-    return NULL;
-  }
-}
-#endif
-
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
   taosRLockLatch(&pMeta->lock);
 
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
-  if (ppTask != NULL && (atomic_load_8(&((*ppTask)->taskStatus)) != TASK_STATUS__DROPPING)) {
+  if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) {
     atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
     taosRUnLockLatch(&pMeta->lock);
     return *ppTask;
@@ -209,7 +197,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1);
   ASSERT(left >= 0);
   if (left == 0) {
-    ASSERT(atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING);
+    ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING);
     tFreeStreamTask(pTask);
   }
 }
@@ -223,7 +211,7 @@ SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId) {
     SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pRestoreTasks, &taskId, sizeof(int32_t));
     if (p != NULL) {
       pTask = *p;
-      if (pTask != NULL && (atomic_load_8(&(pTask->taskStatus)) != TASK_STATUS__DROPPING)) {
+      if (pTask != NULL && (atomic_load_8(&(pTask->status.taskStatus)) != TASK_STATUS__DROPPING)) {
         atomic_add_fetch_32(&pTask->refCnt, 1);
         taosRUnLockLatch(&pMeta->lock);
         return pTask;
@@ -233,7 +221,7 @@ SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId) {
     SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
     if (p != NULL) {
       pTask = *p;
-      if (pTask != NULL && atomic_load_8(&(pTask->taskStatus)) != TASK_STATUS__DROPPING) {
+      if (pTask != NULL && atomic_load_8(&(pTask->status.taskStatus)) != TASK_STATUS__DROPPING) {
         atomic_add_fetch_32(&pTask->refCnt, 1);
         taosRUnLockLatch(&pMeta->lock);
         return pTask;
@@ -255,7 +243,7 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
      * taosTmrStop(pTask->timer);*/
     /*pTask->timer = NULL;*/
     /*}*/
-    atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING);
+    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
 
     taosWLockLatch(&pMeta->lock);
     streamMetaReleaseTask(pMeta, pTask);
@@ -338,9 +326,9 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
       return -1;
     }
 
-    /*pTask->taskStatus = TASK_STATUS__NORMAL;*/
+    /*pTask->status.taskStatus = TASK_STATUS__NORMAL;*/
     if (pTask->fillHistory) {
-      pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
+      pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
       streamTaskCheckDownstream(pTask, ver);
     }
   }
diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c
index 3e7a02b8d5..9962cdfcc0 100644
--- a/source/libs/stream/src/streamRecover.c
+++ b/source/libs/stream/src/streamRecover.c
@@ -18,7 +18,7 @@
 int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
   qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId);
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
-    atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE);
+    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE);
     streamSetParamForRecover(pTask);
     streamSourceRecoverPrepareStep1(pTask, version);
 
@@ -44,11 +44,11 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
     }
 
   } else if (pTask->taskLevel == TASK_LEVEL__AGG) {
-    atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
+    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
     streamSetParamForRecover(pTask);
     streamAggRecoverPrepare(pTask);
   } else if (pTask->taskLevel == TASK_LEVEL__SINK) {
-    atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
+    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
   }
   return 0;
 }
@@ -122,7 +122,7 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp
 }
 
 int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) {
-  return atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL;
+  return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL;
 }
 
 int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) {
@@ -168,7 +168,7 @@ int32_t streamRestoreParam(SStreamTask* pTask) {
   return qStreamRestoreParam(exec);
 }
 int32_t streamSetStatusNormal(SStreamTask* pTask) {
-  atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
+  atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
   return 0;
 }
 
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index f45b6ad7b7..834c022a9a 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -29,7 +29,7 @@ SStreamTask* tNewStreamTask(int64_t streamId) {
   sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId);
 
   pTask->id.idStr = taosStrdup(buf);
-  pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
+  pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
   pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
 
@@ -63,8 +63,8 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1;
   if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1;
 
-  if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->schedStatus) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->status.taskStatus) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->status.schedStatus) < 0) return -1;
 
   if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
@@ -116,8 +116,8 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1;
   if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1;
 
-  if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->schedStatus) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->status.taskStatus) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->status.schedStatus) < 0) return -1;
 
   if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;

From 33143e90c831de915b819f5721417f5dbf17a7df Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Wed, 12 Apr 2023 09:17:31 +0800
Subject: [PATCH 10/65] enhance: add udf stub expiration of 10s after creation

---
 source/libs/function/src/tudf.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 2269ad7f6a..92dbdda496 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -343,7 +343,7 @@ typedef struct SUdfcFuncStub {
   char           udfName[TSDB_FUNC_NAME_LEN + 1];
   UdfcFuncHandle handle;
   int32_t        refCount;
-  int64_t        lastRefTime;
+  int64_t        createTime;
 } SUdfcFuncStub;
 
 typedef struct SUdfcProxy {
@@ -982,15 +982,15 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
   if (stubIndex != -1) {
     SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex);
     UdfcFuncHandle handle = foundStub->handle;
-    if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
+    int64_t currUs = taosGetTimestampUs();
+    if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL && (currUs - foundStub->createTime) < 10 * 1000 * 1000) {
       *pHandle = foundStub->handle;
       ++foundStub->refCount;
-      foundStub->lastRefTime = taosGetTimestampUs();
       uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
       return 0;
     } else {
-      fnInfo("invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", udfName,
-             foundStub->refCount, foundStub->lastRefTime);
+      fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName,
+             foundStub->refCount, foundStub->createTime);
       taosArrayRemove(gUdfcProxy.udfStubs, stubIndex);
     }
   }
@@ -1001,7 +1001,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
     strncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN);
     stub.handle = *pHandle;
     ++stub.refCount;
-    stub.lastRefTime = taosGetTimestampUs();
+    stub.createTime = taosGetTimestampUs();
     taosArrayPush(gUdfcProxy.udfStubs, &stub);
     taosArraySort(gUdfcProxy.udfStubs, compareUdfcFuncSub);
   } else {
@@ -1046,14 +1046,14 @@ int32_t cleanUpUdfs() {
       fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount);
       doTeardownUdf(stub->handle);
     } else {
-      fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %" PRId64 ", handle: %p", stub->udfName,
-             stub->refCount, stub->lastRefTime, stub->handle);
+      fnInfo("udf still in use. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName,
+             stub->refCount, stub->createTime, stub->handle);
       UdfcFuncHandle handle = stub->handle;
       if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
         taosArrayPush(udfStubs, stub);
       } else {
-        fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache",
-               stub->udfName, stub->refCount, stub->lastRefTime);
+        fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache",
+               stub->udfName, stub->refCount, stub->createTime);
       }
     }
     ++i;

From bb22d9ee5e228b92449d2eadf5c7bbc9286cb6af Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 12 Apr 2023 10:56:19 +0800
Subject: [PATCH 11/65] fix:open test cases for tmq & add log if rebalance
 error

---
 source/dnode/mnode/impl/src/mndSubscribe.c | 2 +-
 tests/parallel_test/cases.task             | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index 64a3170d47..2eb5d2a69c 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -670,7 +670,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
     // possibly no vg is changed
     // when each topic is re-balanced, issue an trans to save the results in sdb.
     if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) {
-      mError("mq re-balance persist output error, possibly vnode splitted or dropped");
+      mError("mq re-balance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr());
     }
 
     taosArrayDestroy(rebOutput.newConsumers);
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index d1fbacdadf..c14f316ffc 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -89,12 +89,12 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py
-# ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py
-#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py

From fe718f60ee1120d0d7bd7618d76dd99655974d4d Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Wed, 12 Apr 2023 12:15:09 +0800
Subject: [PATCH 12/65] fix: add expired udfc func stub to track the expired

---
 source/libs/function/src/tudf.c | 54 +++++++++++++++++++++++++++------
 1 file changed, 45 insertions(+), 9 deletions(-)

diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 92dbdda496..53b2cf4aca 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -363,6 +363,7 @@ typedef struct SUdfcProxy {
 
   uv_mutex_t udfStubsMutex;
   SArray    *udfStubs;  // SUdfcFuncStub
+  SArray    *expiredUdfStubs; //SUdfcFuncStub
 
   uv_mutex_t udfcUvMutex;
   int8_t     initialized;
@@ -983,15 +984,22 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
     SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex);
     UdfcFuncHandle handle = foundStub->handle;
     int64_t currUs = taosGetTimestampUs();
-    if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL && (currUs - foundStub->createTime) < 10 * 1000 * 1000) {
-      *pHandle = foundStub->handle;
-      ++foundStub->refCount;
-      uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
-      return 0;
+    bool expired = (currUs - foundStub->createTime) >= 10 * 1000 * 1000;
+    if (!expired) {
+      if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
+        *pHandle = foundStub->handle;
+        ++foundStub->refCount;
+        uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
+        return 0;
+      } else {
+        fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName,
+               foundStub->refCount, foundStub->createTime);
+        taosArrayRemove(gUdfcProxy.udfStubs, stubIndex);
+      }
     } else {
-      fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName,
-             foundStub->refCount, foundStub->createTime);
       taosArrayRemove(gUdfcProxy.udfStubs, stubIndex);
+      taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub);
+      taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub);
     }
   }
   *pHandle = NULL;
@@ -1017,13 +1025,17 @@ void releaseUdfFuncHandle(char *udfName) {
   SUdfcFuncStub key = {0};
   strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
   SUdfcFuncStub *foundStub = taosArraySearch(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
-  if (!foundStub) {
+  SUdfcFuncStub *expiredStub = taosArraySearch(gUdfcProxy.expiredUdfStubs, &key, compareUdfcFuncSub, TD_EQ);
+  if (!foundStub && !expiredStub) {
     uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
     return;
   }
-  if (foundStub->refCount > 0) {
+  if (foundStub != NULL && foundStub->refCount > 0) {
     --foundStub->refCount;
   }
+  if (expiredStub != NULL && expiredStub->refCount > 0) {
+    --expiredStub->refCount;
+  }
   uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
 }
 
@@ -1060,6 +1072,28 @@ int32_t cleanUpUdfs() {
   }
   taosArrayDestroy(gUdfcProxy.udfStubs);
   gUdfcProxy.udfStubs = udfStubs;
+
+  SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
+  while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) {
+    SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i);
+    if (stub->refCount == 0) {
+      fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount);
+      doTeardownUdf(stub->handle);
+    } else {
+      fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName,
+             stub->refCount, stub->createTime, stub->handle);
+      UdfcFuncHandle handle = stub->handle;
+      if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
+        taosArrayPush(expiredUdfStubs, stub);
+      } else {
+        fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache",
+               stub->udfName, stub->refCount, stub->createTime);
+      }
+    }
+    ++i;
+  }
+  taosArrayDestroy(gUdfcProxy.udfStubs);
+  gUdfcProxy.expiredUdfStubs = expiredUdfStubs;
   uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
   return 0;
 }
@@ -1663,6 +1697,7 @@ int32_t udfcOpen() {
   uv_barrier_wait(&proxy->initBarrier);
   uv_mutex_init(&proxy->udfStubsMutex);
   proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub));
+  proxy->expiredUdfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub));
   uv_mutex_init(&proxy->udfcUvMutex);
   fnInfo("udfc initialized") return 0;
 }
@@ -1679,6 +1714,7 @@ int32_t udfcClose() {
   uv_thread_join(&udfc->loopThread);
   uv_mutex_destroy(&udfc->taskQueueMutex);
   uv_barrier_destroy(&udfc->initBarrier);
+  taosArrayDestroy(udfc->expiredUdfStubs);
   taosArrayDestroy(udfc->udfStubs);
   uv_mutex_destroy(&udfc->udfStubsMutex);
   uv_mutex_destroy(&udfc->udfcUvMutex);

From dcf77f25296e359dff623e8f12f3e3ecd6f372f6 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Wed, 12 Apr 2023 14:24:34 +0800
Subject: [PATCH 13/65] fix: fix constant comparision precision error

---
 source/libs/scalar/src/scalar.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index fe01977b2e..21a8930ddf 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -1038,23 +1038,29 @@ bool sclContainsAggFuncNode(SNode *pNode) {
   return aggFunc;
 }
 
+static int8_t sclGetOpValueNodeTsPrecision(SNode *pLeft, SNode *pRight) {
+  int8_t lPrec = ((SExprNode *)pLeft)->resType.precision;
+  int8_t rPrec = ((SExprNode *)pRight)->resType.precision;
+
+  return TMAX(lPrec, rPrec);
+}
 
 int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) {
   int32_t code = 0;
-  
+
   if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) {
     if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) {
-      SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pRight)->resType.precision, (SValueNode*)node->pLeft));
+      SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft));
     }
   } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) {
     if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) {
       if (SCL_IS_VAR_VALUE_NODE(node->pRight)) {
-        SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)node->pRight));
+        SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight));
       } else if (QUERY_NODE_NODE_LIST == node->pRight->type) {
         SNode* pNode;
         FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) {
           if (SCL_IS_VAR_VALUE_NODE(pNode)) {
-            SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)pNode));
+            SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode));
           }
         }
       }
@@ -1077,7 +1083,7 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
   if (NULL == node->pCase) {
     return TSDB_CODE_SUCCESS;
   }
-  
+
   if (SCL_IS_VAR_VALUE_NODE(node->pCase)) {
     SNode* pNode;
     FOREACH(pNode, node->pWhenThenList) {

From 6b20306bef20b75669c01d1ffffb179c072bbfc8 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 12 Apr 2023 17:44:59 +0800
Subject: [PATCH 14/65] test: add flush database in tsim/parser/last_cache.sim
 and limit1.sim

---
 tests/script/tsim/parser/last_cache.sim | 1 +
 tests/script/tsim/parser/limit1.sim     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim
index 9a41a9f5aa..3f1a29d928 100644
--- a/tests/script/tsim/parser/last_cache.sim
+++ b/tests/script/tsim/parser/last_cache.sim
@@ -54,6 +54,7 @@ sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL )
 
 run tsim/parser/last_cache_query.sim
 
+sql flush database $db
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 
 system sh/exec.sh -n dnode1 -s start
diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim
index d1a75f3ba9..bae5eba7a4 100644
--- a/tests/script/tsim/parser/limit1.sim
+++ b/tests/script/tsim/parser/limit1.sim
@@ -55,6 +55,7 @@ print ====== tables created
 run tsim/parser/limit1_tb.sim
 run tsim/parser/limit1_stb.sim
 
+sql flush database $db
 print ================== restart server to commit data into disk
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start

From 2a0493f40cd8d3576b8c02cc5781e380abf9d0a3 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Thu, 13 Apr 2023 09:53:25 +0800
Subject: [PATCH 15/65] enh: refactor some vars in
 syncLogReplProcessReplyAsNormal

---
 source/libs/sync/src/syncPipeline.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c
index 69888ed8ea..6bebef77dc 100644
--- a/source/libs/sync/src/syncPipeline.c
+++ b/source/libs/sync/src/syncPipeline.c
@@ -916,10 +916,10 @@ int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode,
   ASSERT(pMgr->restored == true);
   if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) {
     if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) {
-      int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs;
-      int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs;
-      int64_t timeDiffMs = lastSentMs - firstSentMs;
-      if (timeDiffMs > 0 && timeDiffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) {
+      int64_t firstMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs;
+      int64_t lastMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs;
+      int64_t diffMs = lastMs - firstMs;
+      if (diffMs > 0 && diffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) {
         pMgr->retryBackoff -= 1;
       }
     }

From 7648e03e3131cfd5023c56865c572e657b8f07ac Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 13 Apr 2023 10:03:30 +0800
Subject: [PATCH 16/65] fix:doBitmapMerge error if remaind bytes is not 0

---
 source/common/src/tdatablock.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 0dd8cb9b0c..986a46036b 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -221,6 +221,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
   }
 
   uint8_t* p = (uint8_t*)pSource->nullbitmap;
+  pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] &= (0B11111111 << shiftBits);  // clear remind bits
   pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits);  // copy remind bits
 
   if (BitmapLen(numOfRow1) == BitmapLen(total)) {
@@ -232,6 +233,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
 
   uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)];
   int32_t  overCount = BitmapLen(total) - BitmapLen(numOfRow1);
+  memset(start, 0, overCount);
   while (i < len) {  // size limit of pSource->nullbitmap
     if (i >= 1) {
       start[i - 1] |= (p[i] >> remindBits);  // copy remind bits
@@ -309,8 +311,9 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int
       pColumnInfoData->pData = tmp;
       if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) {
         char*    btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows));
-        uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1);
-        memset(btmp + BitmapLen(numOfRow1), 0, extend);
+        if (btmp == NULL) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
 
         pColumnInfoData->nullbitmap = btmp;
       }

From 7414f28d175dc9745bccb5bedb6ee20deb54be24 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Thu, 13 Apr 2023 11:22:15 +0800
Subject: [PATCH 17/65] fix translatefunc not getting precision issue

---
 include/libs/function/functionMgt.h    |  2 +-
 source/libs/function/src/functionMgt.c |  3 +++
 source/libs/parser/src/parTranslater.c | 14 ++++++++++++++
 source/libs/scalar/src/scalar.c        | 19 +++++++++++++++----
 4 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h
index 60a4a8c605..42bc89f0b7 100644
--- a/include/libs/function/functionMgt.h
+++ b/include/libs/function/functionMgt.h
@@ -189,7 +189,7 @@ bool fmIsScalarFunc(int32_t funcId);
 bool fmIsVectorFunc(int32_t funcId);
 bool fmIsIndefiniteRowsFunc(int32_t funcId);
 bool fmIsStringFunc(int32_t funcId);
-bool fmIsDatetimeFunc(int32_t funcId);
+bool fmIsDateTimeFunc(int32_t funcId);
 bool fmIsSelectFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c
index 94ab616dda..18f6e8050b 100644
--- a/source/libs/function/src/functionMgt.c
+++ b/source/libs/function/src/functionMgt.c
@@ -174,6 +174,8 @@ bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC
 
 bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); }
 
+bool fmIsDateTimeFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_DATETIME_FUNC); }
+
 bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); }
 
 bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); }
@@ -184,6 +186,7 @@ bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWi
 
 bool fmIsIndefiniteRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INDEFINITE_ROWS_FUNC); }
 
+
 bool fmIsSpecialDataRequiredFunc(int32_t funcId) {
   return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED);
 }
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1d6d123cb4..f9fb075149 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1539,6 +1539,16 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateDateTimeFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
+  if (!fmIsDateTimeFunc(pFunc->funcId)) {
+    return TSDB_CODE_SUCCESS;
+  }
+  SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
+  pFunc->node.resType.precision = pSelect->precision;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static bool hasFillClause(SNode* pCurrStmt) {
   if (!isSelectStmt(pCurrStmt)) {
     return false;
@@ -1678,6 +1688,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
     SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt;
     pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId);
     pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId);
+
     if (fmIsIndefiniteRowsFunc(pFunc->funcId)) {
       pSelect->hasIndefiniteRowsFunc = true;
       pSelect->returnRows = fmGetFuncReturnRows(pFunc);
@@ -1822,6 +1833,9 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* p
   if (TSDB_CODE_SUCCESS == code) {
     code = translateTimelineFunc(pCxt, pFunc);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateDateTimeFunc(pCxt, pFunc);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = translateBlockDistFunc(pCxt, pFunc);
   }
diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index 21a8930ddf..0521076d23 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -1038,11 +1038,22 @@ bool sclContainsAggFuncNode(SNode *pNode) {
   return aggFunc;
 }
 
-static int8_t sclGetOpValueNodeTsPrecision(SNode *pLeft, SNode *pRight) {
-  int8_t lPrec = ((SExprNode *)pLeft)->resType.precision;
-  int8_t rPrec = ((SExprNode *)pRight)->resType.precision;
+static uint8_t sclGetOpValueNodeTsPrecision(SNode *pLeft, SNode *pRight) {
+  uint8_t lPrec = ((SExprNode *)pLeft)->resType.precision;
+  uint8_t rPrec = ((SExprNode *)pRight)->resType.precision;
 
-  return TMAX(lPrec, rPrec);
+  uint8_t lType = ((SExprNode *)pLeft)->resType.type;
+  uint8_t rType = ((SExprNode *)pRight)->resType.type;
+
+  if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP == rType) {
+    return TMAX(lPrec, rPrec);
+  } else if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP != rType) {
+    return lPrec;
+  } else if (TSDB_DATA_TYPE_TIMESTAMP == rType && TSDB_DATA_TYPE_TIMESTAMP != lType) {
+    return rPrec;
+  }
+
+  return 0;
 }
 
 int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) {

From ab372d501a05b1512b92dabeb5d487d9451cbed5 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 13 Apr 2023 14:33:37 +0800
Subject: [PATCH 18/65] fix(tsdb/read): release bi cache entry before returning

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index d868c54fd1..627d09b1a9 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -831,6 +831,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader,
       // this block belongs to a table that is not queried.
       STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr);
       if (pScanInfo == NULL) {
+        tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle);
         return terrno;
       }
 

From c7dd22ad1c260f3cf5e5a0716995a553e8aabad8 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Thu, 13 Apr 2023 14:58:13 +0800
Subject: [PATCH 19/65] fix spread timestmap column reading sma issue

---
 source/libs/function/src/builtinsimpl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 6d093c130f..f698834d78 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -3357,7 +3357,7 @@ int32_t spreadFunction(SqlFunctionCtx* pCtx) {
       goto _spread_over;
     }
     double tmin = 0.0, tmax = 0.0;
-    if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) {
       tmin = (double)GET_INT64_VAL(&pAgg->min);
       tmax = (double)GET_INT64_VAL(&pAgg->max);
     } else if (IS_FLOAT_TYPE(type)) {

From a4d818c982f143b87ebfcc948962fdd8a025af8e Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Thu, 13 Apr 2023 15:25:36 +0800
Subject: [PATCH 20/65] fix: alter table check

---
 source/libs/parser/src/parTranslater.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1d6d123cb4..6fd46fc03d 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1692,7 +1692,8 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
     pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType);
     pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType);
     pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType);
-    pSelect->hasInterpPseudoColFunc = pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId);
+    pSelect->hasInterpPseudoColFunc =
+        pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId);
     pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType);
     pSelect->hasLastFunc = pSelect->hasLastFunc ? true : (FUNCTION_TYPE_LAST == pFunc->funcType);
     pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId);
@@ -3372,7 +3373,8 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE);
     }
     if (pSelect->hasInterpPseudoColFunc) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "Has Interp pseudo column(s) but missing interp function");
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
+                                     "Has Interp pseudo column(s) but missing interp function");
     }
     return TSDB_CODE_SUCCESS;
   }
@@ -5159,8 +5161,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
     const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
     if (NULL == pSchema) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
-    } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
-               pSchema->bytes >= calcTypeBytes(pStmt->dataType)) {
+    }
+
+    if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
+        pSchema->bytes >= calcTypeBytes(pStmt->dataType)) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL);
     }
 
@@ -8234,10 +8238,6 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) {
 
 static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
                                      SQuery* pQuery) {
-  if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped");
-  }
-
   if (TSDB_SUPER_TABLE == pTableMeta->tableType) {
     return TSDB_CODE_SUCCESS;
   } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) {

From b51477343d099da315dc51f6a4653054cccd955a Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Thu, 13 Apr 2023 15:50:30 +0800
Subject: [PATCH 21/65] fix crash

---
 source/libs/parser/src/parTranslater.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index f9fb075149..3d6fcbea67 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1543,6 +1543,11 @@ static int32_t translateDateTimeFunc(STranslateContext* pCxt, SFunctionNode* pFu
   if (!fmIsDateTimeFunc(pFunc->funcId)) {
     return TSDB_CODE_SUCCESS;
   }
+
+  if (!isSelectStmt(pCxt->pCurrStmt)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
   pFunc->node.resType.precision = pSelect->precision;
 

From a0be0c1d06f8ee31703c538237f661d2f306c24d Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 13 Apr 2023 16:40:38 +0800
Subject: [PATCH 22/65] fix: add client option tsEnableScience

---
 include/common/tglobal.h      |  1 +
 source/common/src/tglobal.c   |  4 ++++
 tools/shell/src/shellEngine.c | 42 ++++++++++++++++++++++++++++++-----
 3 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 29182ef277..6fde7b48a2 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -112,6 +112,7 @@ extern int32_t tsQueryNodeChunkSize;
 extern bool    tsQueryUseNodeAllocator;
 extern bool    tsKeepColumnName;
 extern bool    tsEnableQueryHb;
+extern bool    tsEnableScience;
 extern int32_t tsRedirectPeriod;
 extern int32_t tsRedirectFactor;
 extern int32_t tsRedirectMaxPeriod;
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index af2da6ae2a..a1c78ac87a 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -105,6 +105,7 @@ int32_t tsQueryPolicy = 1;
 int32_t tsQueryRspPolicy = 0;
 int64_t tsQueryMaxConcurrentTables = 200;  // unit is TSDB_TABLE_NUM_UNIT
 bool    tsEnableQueryHb = false;
+bool    tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true 
 int32_t tsQuerySmaOptimize = 0;
 int32_t tsQueryRsmaTolerance = 1000;  // the tolerance time (ms) to judge from which level to query rsma data.
 bool    tsQueryPlannerTrace = false;
@@ -117,6 +118,7 @@ int32_t tsRedirectMaxPeriod = 1000;
 int32_t tsMaxRetryWaitTime = 10000;
 bool    tsUseAdapter = false;
 
+
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
  * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@@ -328,6 +330,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1;
   if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, false) != 0) return -1;
+  if (cfgAddBool(pCfg, "enableScience", tsEnableScience, false) != 0) return -1;
   if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1;
   if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1;
@@ -730,6 +733,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
   tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
   tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
+  tsScientific =  = cfgGetItem(pCfg, "enableScience")->bval;  
   tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32;
   tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
   tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 11758c76dc..3567382208 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -315,6 +315,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
   quotationStr[0] = '\"';
   quotationStr[1] = 0;
 
+  int  n;
   char buf[TSDB_MAX_BYTES_PER_ROW];
   switch (field->type) {
     case TSDB_DATA_TYPE_BOOL:
@@ -345,11 +346,24 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
       taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val));
       break;
     case TSDB_DATA_TYPE_FLOAT:
-      taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val));
+      if (tsEnableScience) {
+        taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val));
+      } else {
+        taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
+      }
       break;
     case TSDB_DATA_TYPE_DOUBLE:
-      snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", 23, GET_DOUBLE_VAL(val));
-      taosFprintfFile(pFile, "%s", buf);
+      if (tsEnableScience) {
+        snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9e", 23, GET_DOUBLE_VAL(val));
+        taosFprintfFile(pFile, "%s", buf);
+      } else {
+        n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val));
+        if (n > TMAX(25, length)) {
+          taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val));
+        } else {
+          taosFprintfFile(pFile, "%s", buf);
+        }
+      }
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
@@ -492,6 +506,8 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) {
   }
 }
 
+#define FLOAT_NORMAL_MIN  -100000000 
+#define FLOAT_NORMAL_MAX   100000000 
 void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision) {
   if (val == NULL) {
     int32_t w = width;
@@ -505,6 +521,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
     return;
   }
 
+  int  n;
   char buf[TSDB_MAX_BYTES_PER_ROW];
   switch (field->type) {
     case TSDB_DATA_TYPE_BOOL:
@@ -535,11 +552,24 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
       printf("%*" PRIu64, width, *((uint64_t *)val));
       break;
     case TSDB_DATA_TYPE_FLOAT:
-      printf("%*e", width, GET_FLOAT_VAL(val));
+      if (tsEnableScience) {
+        printf("%*e", width, GET_FLOAT_VAL(val));
+      } else {
+        printf("%.5f", GET_FLOAT_VAL(val));
+      }
       break;
     case TSDB_DATA_TYPE_DOUBLE:
-      snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val));
-      printf("%*s", width, buf);
+      if (tsEnableScience) {
+        snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val));
+        printf("%*s", width, buf);
+      } else {
+        n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val));
+        if (n > TMAX(25, length)) {
+            printf("%*.15e", length, GET_DOUBLE_VAL(val));
+        } else {
+            printf("%s", buf);
+        }
+      }
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:

From 73d5bdf01fb41e1739116fa5df3b44b26504b053 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 13 Apr 2023 16:45:06 +0800
Subject: [PATCH 23/65] fix: build error fix

---
 source/common/src/tglobal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index a1c78ac87a..02454d2afe 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -733,7 +733,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
   tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
   tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
-  tsScientific =  = cfgGetItem(pCfg, "enableScience")->bval;  
+  tsEnableScience =  = cfgGetItem(pCfg, "enableScience")->bval;  
   tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32;
   tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
   tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;

From eb03a0bb4824da52828fb5478cbda28fde13536b Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 13 Apr 2023 16:55:22 +0800
Subject: [PATCH 24/65] fix: adjust format

---
 source/common/src/tglobal.c   | 2 +-
 tools/shell/src/shellEngine.c | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 02454d2afe..da4a912238 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -733,7 +733,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
   tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
   tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
-  tsEnableScience =  = cfgGetItem(pCfg, "enableScience")->bval;  
+  tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval;  
   tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32;
   tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
   tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 3567382208..f16ac919b9 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -555,7 +555,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
       if (tsEnableScience) {
         printf("%*e", width, GET_FLOAT_VAL(val));
       } else {
-        printf("%.5f", GET_FLOAT_VAL(val));
+        printf("%*.5f", width, GET_FLOAT_VAL(val));
       }
       break;
     case TSDB_DATA_TYPE_DOUBLE:
@@ -563,9 +563,9 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
         snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val));
         printf("%*s", width, buf);
       } else {
-        n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val));
-        if (n > TMAX(25, length)) {
-            printf("%*.15e", length, GET_DOUBLE_VAL(val));
+        n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val));
+        if (n > TMAX(25, width)) {
+            printf("%*.15e", width, GET_DOUBLE_VAL(val));
         } else {
             printf("%s", buf);
         }

From 1bfeb6669eb672ee7a162e772e29e2b5ac364865 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 13 Apr 2023 17:09:43 +0800
Subject: [PATCH 25/65] fix: remove obsolete code

---
 tools/shell/src/shellEngine.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index f16ac919b9..d13ed0c0a8 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -506,8 +506,6 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) {
   }
 }
 
-#define FLOAT_NORMAL_MIN  -100000000 
-#define FLOAT_NORMAL_MAX   100000000 
 void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision) {
   if (val == NULL) {
     int32_t w = width;

From 17b56695553b6890ba88f3b452576d49691c7e09 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Thu, 13 Apr 2023 17:32:31 +0800
Subject: [PATCH 26/65] fix: optimizing 'alter table drop tag' error reporting

---
 source/libs/parser/src/parTranslater.c | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 7615ef8873..41ae0cef25 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -5155,28 +5155,32 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
   }
 
-  if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) {
-    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped");
-  }
-
   int32_t tagsLen = 0;
   for (int32_t i = 0; i < pTableMeta->tableInfo.numOfTags; ++i) {
     tagsLen += pTagsSchema[i].bytes;
   }
 
   if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType ||
-      TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) {
+      TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType ||
+      TSDB_ALTER_TABLE_DROP_TAG == pStmt->alterType) {
     if (TSDB_SUPER_TABLE != pTableMeta->tableType) {
       return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table");
     }
 
     const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName);
     if (NULL == pSchema) {
-      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName);
+      return generateSyntaxErrMsg(
+          &pCxt->msgBuf,
+          (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType)
+              ? TSDB_CODE_PAR_INVALID_COLUMN
+              : TSDB_CODE_PAR_INVALID_TAG_NAME,
+          pStmt->colName);
     }
 
-    if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
-        pSchema->bytes >= calcTypeBytes(pStmt->dataType)) {
+    if ((TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType ||
+         TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType) &&
+        (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
+         pSchema->bytes >= calcTypeBytes(pStmt->dataType))) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL);
     }
 
@@ -5221,6 +5225,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
     }
   }
 
+  if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped");
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 

From c519c7e4fa84c35611b7fb6bfed3195ac0c40f62 Mon Sep 17 00:00:00 2001
From: Hui Li <52318143+plum-lihui@users.noreply.github.com>
Date: Thu, 13 Apr 2023 17:34:15 +0800
Subject: [PATCH 27/65] Update cases.task

close tmqDelete-1ctb.py
---
 tests/parallel_test/cases.task | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index f6ba91a68c..70f86a4201 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -89,7 +89,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
+#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py

From d8c209b49b1cf3d300974508b4fb557da9ed43b0 Mon Sep 17 00:00:00 2001
From: Hui Li <52318143+plum-lihui@users.noreply.github.com>
Date: Thu, 13 Apr 2023 18:05:16 +0800
Subject: [PATCH 28/65] Update tmqDelete-1ctb.py

---
 tests/system-test/7-tmq/tmqDelete-1ctb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py
index 4b8c8c8629..aa9c8d25d0 100644
--- a/tests/system-test/7-tmq/tmqDelete-1ctb.py
+++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py
@@ -238,7 +238,7 @@ class TDTestCase:
 
         if self.snapshot == 0:
             consumerId     = 2
-            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2
         elif self.snapshot == 1:
             consumerId     = 3
             expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4))
@@ -324,7 +324,7 @@ class TDTestCase:
 
         if self.snapshot == 0:
             consumerId     = 4
-            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2
         elif self.snapshot == 1:
             consumerId     = 5
             expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4))

From b7205d5ccc484d06b1bbbf682b27db02ef5411cc Mon Sep 17 00:00:00 2001
From: Hui Li <52318143+plum-lihui@users.noreply.github.com>
Date: Thu, 13 Apr 2023 18:06:17 +0800
Subject: [PATCH 29/65] Update cases.task

reopen tmqDelete-1ctb.py
---
 tests/parallel_test/cases.task | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 70f86a4201..f6ba91a68c 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -89,7 +89,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py
-#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py

From 06d9a74bcc9ec456f2a8bd8fa03c21443e81e78f Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Thu, 13 Apr 2023 19:38:47 +0800
Subject: [PATCH 30/65] fix: optimizing 'alter table drop tag' error reporting

---
 source/libs/parser/src/parTranslater.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 41ae0cef25..1c731144dc 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -5178,7 +5178,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
     }
 
     if ((TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType ||
-         TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType) &&
+         TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) &&
         (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type ||
          pSchema->bytes >= calcTypeBytes(pStmt->dataType))) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL);

From d8386754eeb6a2411e2f51ab7958d320b073cb97 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Tue, 11 Apr 2023 14:16:31 +0800
Subject: [PATCH 31/65] enh: try to propose vnode commit at vnode closing

---
 source/dnode/mgmt/mgmt_vnode/src/vmInt.c |  6 +++++
 source/dnode/vnode/inc/vnode.h           |  2 +-
 source/dnode/vnode/src/inc/vnd.h         |  3 +--
 source/dnode/vnode/src/vnd/vnodeCommit.c |  5 +++--
 source/dnode/vnode/src/vnd/vnodeSync.c   | 28 ++++++++++++++----------
 5 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index 4c5b1246e7..0244a4fd6e 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
+#include "vnd.h"
 
 SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) {
   SVnodeObj *pVnode = NULL;
@@ -78,6 +79,11 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
 
 void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) {
   char path[TSDB_FILENAME_LEN] = {0};
+  bool atExit = true;
+
+  if (vnodeIsLeader(pVnode->pImpl)) {
+    vnodeProposeCommitOnNeed(pVnode->pImpl, atExit);
+  }
 
   taosThreadRwlockWrlock(&pMgmt->lock);
   taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index a9e5fe628b..7ecfadf728 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -92,7 +92,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo);
 void    vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
 void    vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
-void    vnodeProposeCommitOnNeed(SVnode *pVnode);
+void    vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit);
 
 // meta
 typedef struct SMeta       SMeta;  // todo: remove
diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h
index 134909090f..ae65e2ba3f 100644
--- a/source/dnode/vnode/src/inc/vnd.h
+++ b/source/dnode/vnode/src/inc/vnd.h
@@ -96,7 +96,7 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg);
 
 // vnodeCommit.c
 int32_t vnodeBegin(SVnode* pVnode);
-int32_t vnodeShouldCommit(SVnode* pVnode);
+int32_t vnodeShouldCommit(SVnode* pVnode, bool atExit);
 void    vnodeUpdCommitSched(SVnode* pVnode);
 void    vnodeRollback(SVnode* pVnode);
 int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg);
@@ -115,7 +115,6 @@ void    vnodeSyncClose(SVnode* pVnode);
 void    vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code);
 bool    vnodeIsLeader(SVnode* pVnode);
 bool    vnodeIsRoleLeader(SVnode* pVnode);
-int     vnodeShouldCommit(SVnode* pVnode);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 09f1ca7877..3eb813f394 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -149,7 +149,7 @@ void vnodeUpdCommitSched(SVnode *pVnode) {
   pVnode->commitSched.maxWaitMs = tsVndCommitMaxIntervalMs + (randNum % tsVndCommitMaxIntervalMs);
 }
 
-int vnodeShouldCommit(SVnode *pVnode) {
+int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
   SVCommitSched *pSched = &pVnode->commitSched;
   int64_t        nowMs = taosGetMonoTimestampMs();
   bool           diskAvail = osDataSpaceAvailable();
@@ -158,7 +158,8 @@ int vnodeShouldCommit(SVnode *pVnode) {
   taosThreadMutexLock(&pVnode->mutex);
   if (pVnode->inUse && diskAvail) {
     needCommit =
-        ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs));
+        ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) ||
+        ((pVnode->inUse->size > 0) && atExit);
   }
   taosThreadMutexUnlock(&pVnode->mutex);
   return needCommit;
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index d681f5b65e..2b4eff08d3 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -129,8 +129,8 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak
   return code;
 }
 
-void vnodeProposeCommitOnNeed(SVnode *pVnode) {
-  if (!vnodeShouldCommit(pVnode)) {
+void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit) {
+  if (!vnodeShouldCommit(pVnode, atExit)) {
     return;
   }
 
@@ -145,18 +145,20 @@ void vnodeProposeCommitOnNeed(SVnode *pVnode) {
   rpcMsg.pCont = pHead;
   rpcMsg.info.noResp = 1;
 
+  vInfo("vgId:%d, propose vnode commit", pVnode->config.vgId);
   bool isWeak = false;
-  if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) {
-    vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr());
-    goto _out;
+
+  if (!atExit) {
+    if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) {
+      vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr());
+    }
+    rpcFreeCont(rpcMsg.pCont);
+    rpcMsg.pCont = NULL;
+  } else {
+    tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg);
   }
 
-  vInfo("vgId:%d, proposed vnode commit", pVnode->config.vgId);
-
-_out:
   vnodeUpdCommitSched(pVnode);
-  rpcFreeCont(rpcMsg.pCont);
-  rpcMsg.pCont = NULL;
 }
 
 #if BATCH_ENABLE
@@ -236,7 +238,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
       continue;
     }
 
-    vnodeProposeCommitOnNeed(pVnode);
+    bool atExit = false;
+    vnodeProposeCommitOnNeed(pVnode, atExit);
 
     code = vnodePreProcessWriteMsg(pVnode, pMsg);
     if (code != 0) {
@@ -288,7 +291,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
       continue;
     }
 
-    vnodeProposeCommitOnNeed(pVnode);
+    bool atExit = false;
+    vnodeProposeCommitOnNeed(pVnode, atExit);
 
     code = vnodePreProcessWriteMsg(pVnode, pMsg);
     if (code != 0) {

From 75c7cbdbbb2cb2fb9c0d9fce51b5b28f103c069b Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Thu, 13 Apr 2023 19:58:24 +0800
Subject: [PATCH 32/65] fix: add three more stars for shell mask for main
 (#20916)

---
 compile_flags.txt                |  9 +++++++++
 tools/shell/src/shellArguments.c | 18 +++++++++++-------
 tools/shell/src/shellWebsocket.c |  2 +-
 3 files changed, 21 insertions(+), 8 deletions(-)
 create mode 100644 compile_flags.txt

diff --git a/compile_flags.txt b/compile_flags.txt
new file mode 100644
index 0000000000..c61f9701ab
--- /dev/null
+++ b/compile_flags.txt
@@ -0,0 +1,9 @@
+-DLINUX
+-DWEBSOCKET
+-I/usr/include
+-Iinclude
+-Iinclude/os
+-Iinclude/common
+-Iinclude/util
+-Iinclude/libs/transport
+-Itools/shell/inc
diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c
index 8300e2e1e3..52cb524e3b 100644
--- a/tools/shell/src/shellArguments.c
+++ b/tools/shell/src/shellArguments.c
@@ -243,8 +243,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
   SShellArgs *pArgs = &shell.args;
 
   for (int i = 1; i < argc; i++) {
-    if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 ||
-        strcmp(argv[i], "/?") == 0) {
+    if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0
+            || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) {
       shellParseSingleOpt('?', NULL);
       return 0;
     }
@@ -260,8 +260,10 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
       return -1;
     }
 
-    if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' ||
-        key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N'
+    if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u'
+            || key[1] == 'a' || key[1] == 'c' || key[1] == 's'
+            || key[1] == 'f' || key[1] == 'd' || key[1] == 'w'
+            || key[1] == 'n' || key[1] == 'l' || key[1] == 'N'
 #ifdef WEBSOCKET
         || key[1] == 'E' || key[1] == 'T'
 #endif
@@ -277,10 +279,12 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
       }
       shellParseSingleOpt(key[1], val);
       i++;
-    } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' ||
-               key[1] == 'V' || key[1] == '?' || key[1] == 1
+    } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C'
+                || key[1] == 'r' || key[1] == 'k'
+                || key[1] == 't' || key[1] == 'V'
+                || key[1] == '?' || key[1] == 1
 #ifdef WEBSOCKET
-               || key[1] == 'R'
+            ||key[1] == 'R'
 #endif
     ) {
       shellParseSingleOpt(key[1], NULL);
diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c
index 1d81ce4b2f..d8920cb4c3 100644
--- a/tools/shell/src/shellWebsocket.c
+++ b/tools/shell/src/shellWebsocket.c
@@ -24,7 +24,7 @@ int shell_conn_ws_server(bool first) {
            ((dsnLen-SHELL_WS_DSN_MASK) > SHELL_WS_DSN_BUFF)?
             SHELL_WS_DSN_BUFF:(dsnLen-SHELL_WS_DSN_MASK),
            "%s", shell.args.dsn);
-  fprintf(stdout, "trying to connect %s*** ", cuttedDsn);
+  fprintf(stdout, "trying to connect %s****** ", cuttedDsn);
   fflush(stdout);
   for (int i = 0; i < shell.args.timeout; i++) {
     shell.ws_conn = ws_connect_with_dsn(shell.args.dsn);

From 70b45a4cf035e18023002d3bc3157f26eea818d5 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 13 Apr 2023 23:22:09 +0800
Subject: [PATCH 33/65] fix(stream): all data should be extracted from wal.

---
 include/libs/stream/tstream.h           |  18 ++-
 include/libs/wal/wal.h                  |   2 +-
 source/dnode/snode/src/snode.c          |   2 +-
 source/dnode/vnode/inc/vnode.h          |   3 +-
 source/dnode/vnode/src/inc/tq.h         |   3 +-
 source/dnode/vnode/src/inc/vnodeInt.h   |   2 +-
 source/dnode/vnode/src/tq/tq.c          | 133 ++++++++++----------
 source/dnode/vnode/src/tq/tqPush.c      |  12 +-
 source/dnode/vnode/src/tq/tqRead.c      |  27 +++-
 source/dnode/vnode/src/tq/tqRestore.c   | 161 +++++++++++++++---------
 source/dnode/vnode/src/tq/tqUtil.c      |   5 +-
 source/dnode/vnode/src/vnd/vnodeSvr.c   |   3 -
 source/dnode/vnode/src/vnd/vnodeSync.c  |   2 +-
 source/libs/stream/src/stream.c         |  42 ++++---
 source/libs/stream/src/streamData.c     |   1 -
 source/libs/stream/src/streamDispatch.c |  19 +--
 source/libs/stream/src/streamExec.c     |  26 ++--
 source/libs/stream/src/streamMeta.c     |  63 ++++++----
 source/libs/stream/src/streamTask.c     |   7 +-
 source/util/src/tworker.c               |   3 +-
 tests/script/tsim/stream/basic1.sim     |   4 +-
 21 files changed, 326 insertions(+), 212 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 9f7d366a46..eea8868b8c 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -31,6 +31,7 @@ extern "C" {
 #ifndef _STREAM_H_
 #define _STREAM_H_
 
+typedef void  (*_free_reader_fn_t)(void*);
 typedef struct SStreamTask SStreamTask;
 
 enum {
@@ -218,9 +219,10 @@ void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit);
 SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit);
 
 typedef struct {
-  char*             qmsg;
-  void*             pExecutor;   // not applicable to encoder and decoder
-  struct STqReader* pTqReader;  // not applicable to encoder and decoder
+  char*              qmsg;
+  void*              pExecutor;   // not applicable to encoder and decoder
+  struct STqReader*  pTqReader;   // not applicable to encoder and decoder
+  struct SWalReader* pWalReader;  // not applicable to encoder and decoder
 } STaskExec;
 
 typedef struct {
@@ -331,6 +333,7 @@ struct SStreamTask {
   int64_t             checkpointingId;
   int32_t             checkpointAlignCnt;
   struct SStreamMeta* pMeta;
+  _free_reader_fn_t   freeFp;
 };
 
 // meta
@@ -340,12 +343,14 @@ typedef struct SStreamMeta {
     TTB*         pTaskDb;
     TTB*         pCheckpointDb;
     SHashObj*    pTasks;
-    SHashObj*    pRestoreTasks;
+    SHashObj*    pWalReadTasks;
     void*        ahandle;
     TXN*         txn;
     FTaskExpand* expandFunc;
     int32_t      vgId;
     SRWLatch     lock;
+    int8_t       walScan;
+    bool         quit;
 } SStreamMeta;
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -355,7 +360,7 @@ SStreamTask* tNewStreamTask(int64_t streamId);
 int32_t      tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
 int32_t      tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
 void         tFreeStreamTask(SStreamTask* pTask);
-int32_t      tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem);
+int32_t      tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem);
 bool         tInputQueueIsFull(const SStreamTask* pTask);
 
 static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) {
@@ -568,8 +573,9 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
 void         streamMetaClose(SStreamMeta* streamMeta);
 
 int32_t      streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask);
-int32_t      streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
+int32_t      streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
 int32_t      streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen);
+int32_t      streamMetaGetNumOfTasks(const SStreamMeta* pMeta);
 
 SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index 0a359bfd42..b51289de5e 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -139,7 +139,7 @@ typedef struct {
 } SWalFilterCond;
 
 // todo hide this struct
-typedef struct {
+typedef struct SWalReader {
   SWal          *pWal;
   int64_t        readerId;
   TdFilePtr      pLogFile;
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 7ccbb3b586..4235548e48 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -150,7 +150,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
   ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
 
   // 2.save task
-  code = streamMetaAddTask(pSnode->pMeta, -1, pTask);
+  code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask);
   if (code < 0) {
     return -1;
   }
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index d62eebd2e1..e6e21e1e4a 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -260,7 +260,8 @@ int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
 int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
 
 int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
-void tqNextBlock(STqReader *pReader, SFetchRet *ret);
+void    tqNextBlock(STqReader *pReader, SFetchRet *ret);
+int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData);
 
 int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
 // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver);
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index 94ba399a0a..c007f84790 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -176,7 +176,7 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
 
 // tqStream
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
-int32_t tqDoRestoreSourceStreamTasks(STQ* pTq);
+int32_t tqStreamTasksScanWal(STQ* pTq);
 
 // tq util
 void    createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId);
@@ -187,6 +187,7 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ
 void    doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver);
 void    saveOffsetForAllTasks(STQ* pTq, int64_t ver);
 void    initOffsetForAllRestoreTasks(STQ* pTq);
+int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 412c2549b5..16dea8aebd 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -194,7 +194,7 @@ void    tqClose(STQ*);
 int     tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
 int     tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type);
 int     tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
-int     tqRestoreStreamTasks(STQ* pTq);    // restore all stream tasks after vnode launching completed.
+int     tqStartStreamTasks(STQ* pTq);    // restore all stream tasks after vnode launching completed.
 
 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index b029afc935..2b911befcc 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -15,7 +15,7 @@
 
 #include "tq.h"
 
-#define ALL_STREAM_TASKS_ID       (-1)
+#define WAL_READ_TASKS_ID       (-1)
 
 int32_t tqInit() {
   int8_t old;
@@ -630,6 +630,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
       return -1;
     }
 
+    pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
+
+    pTask->freeFp = (_free_reader_fn_t)tqCloseReader;
     SArray* pList = qGetQueriedTableListInfo(pTask->exec.pExecutor);
     tqReaderAddTbUidList(pTask->exec.pTqReader, pList);
   }
@@ -640,6 +643,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   return 0;
 }
 
+void tFreeStreamTask(SStreamTask* pTask);
+
 int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
   char*               msgStr = pMsg->pCont;
   char*               msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
@@ -754,8 +759,10 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
   tDecoderClear(&decoder);
 
   // 2.save task
-  code = streamMetaAddTask(pTq->pStreamMeta, sversion, pTask);
+  code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask);
   if (code < 0) {
+    tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr,
+            streamMetaGetNumOfTasks(pTq->pStreamMeta));
     return -1;
   }
 
@@ -764,6 +771,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
     streamTaskCheckDownstream(pTask, sversion);
   }
 
+  tqDebug("vgId:%d s-task:%s is deployed from mnd, status:%d, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr,
+          pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta));
   return 0;
 }
 
@@ -973,7 +982,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
       pRefBlock->dataRef = pRef;
       atomic_add_fetch_32(pRefBlock->dataRef, 1);
 
-      if (tAppendDataForStream(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
+      if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
         qError("stream task input del failed, task id %d", pTask->id.taskId);
 
         atomic_sub_fetch_32(pRef, 1);
@@ -1008,7 +1017,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
     taosArrayPush(pStreamBlock->blocks, &block);
 
     if (!failed) {
-      if (tAppendDataForStream(pTask, (SStreamQueueItem*)pStreamBlock) < 0) {
+      if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pStreamBlock) < 0) {
         qError("stream task input del failed, task id %d", pTask->id.taskId);
         continue;
       }
@@ -1036,12 +1045,13 @@ static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTa
   if (code == TSDB_CODE_SUCCESS) {
     tqOffsetDelete(pOffsetStore, key);
   }
-  return TSDB_CODE_SUCCESS;
+
+  return code;
 }
 
 int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
+#if 0
   void* pIter = NULL;
-
   SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit, STREAM_INPUT__DATA_SUBMIT);
   if (pSubmit == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -1050,6 +1060,8 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
     return -1;
   }
 
+  SArray* pInputQueueFullTasks = taosArrayInit(4, POINTER_BYTES);
+
   while (1) {
     pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
     if (pIter == NULL) {
@@ -1081,47 +1093,23 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
         ver = pOffset->val.version;
       }
 
-      tqDebug("s-task:%s input queue is full, do nothing, start ver:%" PRId64, pTask->id.idStr, ver);
+      tqDebug("s-task:%s input queue is full, discard submit block, ver:%" PRId64, pTask->id.idStr, ver);
+      taosArrayPush(pInputQueueFullTasks, &pTask);
       continue;
     }
 
     // check if offset value exists
     STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
-    if (pOffset != NULL) {
-      // seek the stored version and extract data from WAL
-      int32_t code = tqSeekVer(pTask->exec.pTqReader, pOffset->val.version, "");
+    ASSERT(pOffset == NULL);
 
-      // all data has been retrieved from WAL, let's try submit block directly.
-      if (code == TSDB_CODE_SUCCESS) {  // all data retrieved, abort
-        // append the data for the stream
-        SFetchRet ret = {.data.info.type = STREAM_NORMAL};
-        terrno = 0;
-
-        tqNextBlock(pTask->exec.pTqReader, &ret);
-        if (ret.fetchType == FETCH_TYPE__DATA) {
-          code = launchTaskForWalBlock(pTask, &ret, pOffset);
-          if (code != TSDB_CODE_SUCCESS) {
-            continue;
-          }
-        } else {  // FETCH_TYPE__NONE, let's try submit block directly
-          tqDebug("s-task:%s data in WAL are all consumed, try data in submit message", pTask->id.idStr);
-          addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
-        }
-
-        // do nothing if failed, since the offset value is kept already
-      } else {  // failed to seek to the WAL version
-        // todo handle the case where offset has been deleted in WAL, due to stream computing too slow
-        tqDebug("s-task:%s data in WAL are all consumed, try data in submit msg", pTask->id.idStr);
-        addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
-      }
-    } else {
-      addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
-    }
+    addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
   }
 
   streamDataSubmitDestroy(pSubmit);
   taosFreeQitem(pSubmit);
+#endif
 
+  tqStartStreamTasks(pTq);
   return 0;
 }
 
@@ -1131,29 +1119,31 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   int32_t taskId = pReq->taskId;
   int32_t vgId = TD_VID(pTq->pVnode);
 
-  if (taskId == ALL_STREAM_TASKS_ID) {  // all tasks are restored from the wal
-    tqDoRestoreSourceStreamTasks(pTq);
+  if (taskId == WAL_READ_TASKS_ID) {  // all tasks are extracted submit data from the wal
+    tqStreamTasksScanWal(pTq);
+    return 0;
+  }
+
+  SStreamTask* pTask = streamMetaAcquireTaskEx(pTq->pStreamMeta, taskId);
+  if (pTask != NULL) {
+    if (pTask->status.taskStatus == TASK_STATUS__NORMAL) {
+      tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
+      streamProcessRunReq(pTask);
+    } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) {
+      tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId,
+              pTask->id.idStr, pTask->chkInfo.version);
+      streamProcessRunReq(pTask);
+    } else {
+      tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr);
+    }
+
+    streamMetaReleaseTask(pTq->pStreamMeta, pTask);
+
+    tqStartStreamTasks(pTq);
     return 0;
   } else {
-    SStreamTask* pTask = streamMetaAcquireTaskEx(pTq->pStreamMeta, taskId);
-    if (pTask != NULL) {
-      if (pTask->status.taskStatus == TASK_STATUS__NORMAL) {
-        tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
-        streamProcessRunReq(pTask);
-      } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) {
-        tqDebug("vgId:%d s-task:%s start to process in restore procedure from last chk point:%" PRId64, vgId,
-                pTask->id.idStr, pTask->chkInfo.version);
-        streamProcessRunReq(pTask);
-      } else {
-        tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr);
-      }
-
-      streamMetaReleaseTask(pTq->pStreamMeta, pTask);
-      return 0;
-    } else {
-      tqError("vgId:%d failed to found s-task, taskId:%d", vgId, taskId);
-      return -1;
-    }
+    tqError("vgId:%d failed to found s-task, taskId:%d", vgId, taskId);
+    return -1;
   }
 }
 
@@ -1165,14 +1155,10 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
   SDecoder           decoder;
   tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
   tDecodeStreamDispatchReq(&decoder, &req);
-  int32_t taskId = req.taskId;
 
-  SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
+  SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId);
   if (pTask) {
-    SRpcMsg rsp = {
-        .info = pMsg->info,
-        .code = 0,
-    };
+    SRpcMsg rsp = { .info = pMsg->info, .code = 0 };
     streamProcessDispatchReq(pTask, &req, &rsp, exec);
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     return 0;
@@ -1294,26 +1280,39 @@ FAIL:
 
 int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; }
 
-int32_t tqRestoreStreamTasks(STQ* pTq) {
+int32_t tqStartStreamTasks(STQ* pTq) {
   int32_t vgId = TD_VID(pTq->pVnode);
 
+  SStreamMeta* pMeta = pTq->pStreamMeta;
+  taosWLockLatch(&pMeta->lock);
+  pMeta->walScan += 1;
+
+  if (pMeta->walScan > 1) {
+    tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScan);
+    taosWUnLockLatch(&pTq->pStreamMeta->lock);
+    return 0;
+  }
+
   SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
   if (pRunReq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr(terrno));
+    taosWUnLockLatch(&pTq->pStreamMeta->lock);
     return -1;
   }
 
-  int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pRestoreTasks);
-  tqInfo("vgId:%d start restoring stream tasks, total tasks:%d", vgId, numOfTasks);
+  int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks);
+
+  tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
   initOffsetForAllRestoreTasks(pTq);
 
   pRunReq->head.vgId = vgId;
   pRunReq->streamId = 0;
-  pRunReq->taskId = ALL_STREAM_TASKS_ID;
+  pRunReq->taskId = WAL_READ_TASKS_ID;
 
   SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
   tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg);
+  taosWUnLockLatch(&pTq->pStreamMeta->lock);
 
   return 0;
 }
diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c
index d651e945b5..df6648a6af 100644
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@@ -322,16 +322,19 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
     taosWUnLockLatch(&pTq->lock);
   }
 
+  tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, (int)taosHashGetSize(pTq->pStreamMeta->pTasks));
+
   // push data for stream processing:
-  // 1. the vnode isn't in the restore procedure.
+  // 1. the vnode has already been restored.
   // 2. the vnode should be the leader.
   // 3. the stream is not suspended yet.
-  if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && (!pTq->pVnode->restored)) {
+  if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && pTq->pVnode->restored) {
     if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) {
       return 0;
     }
 
     if (msgType == TDMT_VND_SUBMIT) {
+#if 0
       void* data = taosMemoryMalloc(len);
       if (data == NULL) {
         // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry
@@ -343,7 +346,10 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
       memcpy(data, pReq, len);
       SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver};
 
-      tqDebug("tq copy submit msg:%p len:%d ver:%" PRId64 " from %p for stream", data, len, ver, pReq);
+      tqDebug("vgId:%d tq copy submit msg:%p len:%d ver:%" PRId64 " from %p for stream", vgId, data, len, ver, pReq);
+      tqProcessSubmitReq(pTq, submit);
+#endif
+      SPackedData submit = {0};
       tqProcessSubmitReq(pTq, submit);
     }
 
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 14d599551d..69624f4d10 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -300,6 +300,28 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
   return 0;
 }
 
+int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) {
+  if (walNextValidMsg(pReader) < 0) {
+    return -1;
+  }
+
+  void*   pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg));
+  int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
+  int64_t ver = pReader->pHead->head.version;
+
+  void* data = taosMemoryMalloc(len);
+  if (data == NULL) {
+    // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0);
+    return -1;
+  }
+
+  memcpy(data, pBody, len);
+  *pPackedData = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data};
+  return 0;
+}
+
 void tqNextBlock(STqReader* pReader, SFetchRet* ret) {
   while (1) {
     if (pReader->msg2.msgStr == NULL) {
@@ -434,7 +456,10 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD
   SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
   pReader->nextBlk++;
 
-  if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData;
+  if (pSubmitTbDataRet) {
+    *pSubmitTbDataRet = pSubmitTbData;
+  }
+
   int32_t sversion = pSubmitTbData->sver;
   int64_t suid = pSubmitTbData->suid;
   int64_t uid = pSubmitTbData->uid;
diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c
index 877c686d35..6ed74ddcc3 100644
--- a/source/dnode/vnode/src/tq/tqRestore.c
+++ b/source/dnode/vnode/src/tq/tqRestore.c
@@ -15,60 +15,81 @@
 
 #include "tq.h"
 
-static int32_t restoreStreamTaskImpl(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList);
+static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle);
 static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
 
 // this function should be executed by stream threads.
 // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure
 // will not stop eventually.
-int tqDoRestoreSourceStreamTasks(STQ* pTq) {
+int tqStreamTasksScanWal(STQ* pTq) {
+  int32_t vgId = TD_VID(pTq->pVnode);
+  SStreamMeta* pMeta = pTq->pStreamMeta;
   int64_t st = taosGetTimestampMs();
+
   while (1) {
-    SArray* pTaskList = taosArrayInit(4, POINTER_BYTES);
+    tqInfo("vgId:%d continue check if data in wal are available", vgId);
 
     // check all restore tasks
-    restoreStreamTaskImpl(pTq->pStreamMeta, pTq->pOffsetStore, pTaskList);
-    transferToNormalTask(pTq->pStreamMeta, pTaskList);
-    taosArrayDestroy(pTaskList);
+    bool allFull = true;
+    streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &allFull);
 
-    int32_t numOfRestored = taosHashGetSize(pTq->pStreamMeta->pRestoreTasks);
-    if (numOfRestored <= 0) {
-      break;
+    int32_t times = 0;
+
+    if (allFull) {
+      taosWLockLatch(&pMeta->lock);
+      pMeta->walScan -= 1;
+      times = pMeta->walScan;
+
+      if (pMeta->walScan <= 0) {
+        taosWUnLockLatch(&pMeta->lock);
+        break;
+      }
+
+      taosWUnLockLatch(&pMeta->lock);
+      tqInfo("vgId:%d scan wal for stream tasks for %d times", vgId, times);
     }
   }
 
-  int64_t et = taosGetTimestampMs();
-  tqInfo("vgId:%d restoring task completed, elapsed time:%" PRId64 " sec.", TD_VID(pTq->pVnode), (et - st));
+  double el = (taosGetTimestampMs() - st) / 1000.0;
+  tqInfo("vgId:%d scan wal for stream tasks completed, elapsed time:%.2f sec", vgId, el);
+
+  // restore wal scan flag
+//  atomic_store_8(&pTq->pStreamMeta->walScan, 0);
   return 0;
 }
 
-int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
-  int32_t numOfTask = taosArrayGetSize(pTaskList);
-  if (numOfTask <= 0)  {
-    return TSDB_CODE_SUCCESS;
-  }
+//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
+//  int32_t numOfTask = taosArrayGetSize(pTaskList);
+//  if (numOfTask <= 0)  {
+//    return TSDB_CODE_SUCCESS;
+//  }
+//
+//  // todo: add lock
+//  for (int32_t i = 0; i < numOfTask; ++i) {
+//    SStreamTask* pTask = taosArrayGetP(pTaskList, i);
+//    tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
+//            pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
+//    taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
+//
+//    // NOTE: do not change the following order
+//    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
+//    taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
+//  }
+//
+//  return TSDB_CODE_SUCCESS;
+//}
 
-  // todo: add lock
-  for (int32_t i = 0; i < numOfTask; ++i) {
-    SStreamTask* pTask = taosArrayGetP(pTaskList, i);
-    tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
-            pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
-    taosHashRemove(pStreamMeta->pRestoreTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
+int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) {
+  void*   pIter = NULL;
+  int32_t vgId = pStreamMeta->vgId;
 
-    // NOTE: do not change the following order
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
-    taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
-  }
+  *pScanIdle = true;
 
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t restoreStreamTaskImpl(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, SArray* pTaskList) {
-  // check all restore tasks
-  void* pIter = NULL;
+  bool allWalChecked = true;
+  tqDebug("vgId:%d start to check wal to extract new submit block", vgId);
 
   while (1) {
-    pIter = taosHashIterate(pStreamMeta->pRestoreTasks, pIter);
+    pIter = taosHashIterate(pStreamMeta->pTasks, pIter);
     if (pIter == NULL) {
       break;
     }
@@ -78,8 +99,10 @@ int32_t restoreStreamTaskImpl(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS
       continue;
     }
 
-    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, pTask->status.taskStatus);
+    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE ||
+        pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
+              pTask->status.taskStatus);
       continue;
     }
 
@@ -88,41 +111,57 @@ int32_t restoreStreamTaskImpl(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS
     createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
 
     if (tInputQueueIsFull(pTask)) {
-      tqDebug("s-task:%s input queue is full, do nothing", pTask->id.idStr);
-      taosMsleep(10);
+      tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr);
       continue;
     }
 
+    *pScanIdle = false;
+
     // check if offset value exists
     STqOffset* pOffset = tqOffsetRead(pOffsetStore, key);
-    if (pOffset != NULL) {
-      // seek the stored version and extract data from WAL
-      int32_t code = tqSeekVer(pTask->exec.pTqReader, pOffset->val.version, "");
-      if (code == TSDB_CODE_SUCCESS) {  // all data retrieved, abort
-        // append the data for the stream
-        SFetchRet ret = {.data.info.type = STREAM_NORMAL};
-        terrno = 0;
+    ASSERT(pOffset != NULL);
 
-        tqNextBlock(pTask->exec.pTqReader, &ret);
-        if (ret.fetchType == FETCH_TYPE__DATA) {
-          code = launchTaskForWalBlock(pTask, &ret, pOffset);
-          if (code != TSDB_CODE_SUCCESS) {
-            continue;
-          }
-        } else {
-          // FETCH_TYPE__NONE: all data has been retrieved from WAL, let's try submit block directly.
-          tqDebug("s-task:%s data in WAL are all consumed, transfer this task to be normal state", pTask->id.idStr);
-          taosArrayPush(pTaskList, &pTask);
-        }
-      } else {  // failed to seek to the WAL version
-        tqDebug("s-task:%s data in WAL are all consumed, transfer this task to be normal state", pTask->id.idStr);
-        taosArrayPush(pTaskList, &pTask);
-      }
-    } else {
-      ASSERT(0);
+    // seek the stored version and extract data from WAL
+    int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version);
+    if (code != TSDB_CODE_SUCCESS) {  // no data in wal, quit
+      continue;
     }
+
+    // append the data for the stream
+    tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr);
+
+    SPackedData packData = {0};
+    code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData);
+    if (code != TSDB_CODE_SUCCESS) {  // failed, continue
+      continue;
+    }
+
+    SStreamDataSubmit2* p = streamDataSubmitNew(packData, STREAM_INPUT__DATA_SUBMIT);
+    if (p == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr);
+      continue;
+    }
+
+    allWalChecked = false;
+
+    tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr);
+    code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver);
+    if (code == TSDB_CODE_SUCCESS) {
+      pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader);
+      tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
+              pOffset->val.version);
+    } else {
+      // do nothing
+    }
+
+    streamDataSubmitDestroy(p);
+    taosFreeQitem(p);
   }
 
+  if (allWalChecked) {
+    *pScanIdle = true;
+  }
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 2e8c6a53bb..791bfbe6df 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -35,7 +35,7 @@ void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
 }
 
 int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
-  int32_t code = tAppendDataForStream(pTask, pQueueItem);
+  int32_t code = tAppendDataToInputQueue(pTask, pQueueItem);
   if (code < 0) {
     tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver);
     return -1;
@@ -79,7 +79,7 @@ void initOffsetForAllRestoreTasks(STQ* pTq) {
   void* pIter = NULL;
 
   while(1) {
-    pIter = taosHashIterate(pTq->pStreamMeta->pRestoreTasks, pIter);
+    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
     if (pIter == NULL) {
       break;
     }
@@ -103,7 +103,6 @@ void initOffsetForAllRestoreTasks(STQ* pTq) {
       doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version);
     }
   }
-
 }
 
 void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 76ff04b81a..24ebbe23d2 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -539,13 +539,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
       return vnodeGetBatchMeta(pVnode, pMsg);
     case TDMT_VND_TMQ_CONSUME:
       return tqProcessPollReq(pVnode->pTq, pMsg);
-
     case TDMT_STREAM_TASK_RUN:
       return tqProcessTaskRunReq(pVnode->pTq, pMsg);
-#if 1
     case TDMT_STREAM_TASK_DISPATCH:
       return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true);
-#endif
     case TDMT_STREAM_TASK_CHECK:
       return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_DISPATCH_RSP:
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index eb3c5d1f64..9f5d722583 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -551,7 +551,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
   vInfo("vgId:%d, sync restore finished", pVnode->config.vgId);
 
   // start to restore all stream tasks
-  tqRestoreStreamTasks(pVnode->pTq);
+  tqStartStreamTasks(pVnode->pTq);
 }
 
 static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 71d4e5efd8..5ec5be169e 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -70,7 +70,7 @@ void streamSchedByTimer(void* param, void* tmrId) {
     trigger->pBlock->info.type = STREAM_GET_ALL;
     atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE);
 
-    if (tAppendDataForStream(pTask, (SStreamQueueItem*)trigger) < 0) {
+    if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)trigger) < 0) {
       taosFreeQitem(trigger);
       taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer);
       return;
@@ -110,16 +110,17 @@ int32_t streamSchedExec(SStreamTask* pTask) {
 
     SRpcMsg msg = { .msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq) };
     tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg);
+    qDebug("trigger to run s-task:%s", pTask->id.idStr);
   }
 
   return 0;
 }
 
-int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
+int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
   SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
   int8_t            status;
 
-  // enqueue
+  // enqueue data block
   if (pData != NULL) {
     pData->type = STREAM_INPUT__DATA_BLOCK;
     pData->srcVgId = pReq->dataSrcVgId;
@@ -127,10 +128,10 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR
     /*pData->blocks = pReq->data;*/
     /*pBlock->sourceVer = pReq->sourceVer;*/
     streamDispatchReqToData(pReq, pData);
-    if (tAppendDataForStream(pTask, (SStreamQueueItem*)pData) == 0) {
+    if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) {
       status = TASK_INPUT_STATUS__NORMAL;
-    } else {
-      status = TASK_INPUT_STATUS__FAILED;
+    } else {  // input queue is full, upstream is blocked now
+      status = TASK_INPUT_STATUS__BLOCKED;
     }
   } else {
     streamTaskInputFail(pTask);
@@ -148,8 +149,10 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR
   pCont->downstreamNodeId = htonl(pTask->nodeId);
   pCont->downstreamTaskId = htonl(pTask->id.taskId);
   pRsp->pCont = buf;
+
   pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
   tmsgSendRsp(pRsp);
+
   return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
 }
 
@@ -168,7 +171,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
     /*pData->blocks = pReq->data;*/
     /*pBlock->sourceVer = pReq->sourceVer;*/
     streamRetrieveReqToData(pReq, pData);
-    if (tAppendDataForStream(pTask, (SStreamQueueItem*)pData) == 0) {
+    if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) {
       status = TASK_INPUT_STATUS__NORMAL;
     } else {
       status = TASK_INPUT_STATUS__FAILED;
@@ -209,10 +212,10 @@ int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
 }
 
 int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) {
-  qDebug("task %d receive dispatch req from node %d task %d", pTask->id.taskId, pReq->upstreamNodeId,
+  qDebug("vgId:%d s-task:%s receive dispatch req from taskId:%d", pReq->upstreamNodeId, pTask->id.idStr,
          pReq->upstreamTaskId);
 
-  streamTaskEnqueue(pTask, pReq, pRsp);
+  streamTaskEnqueueBlocks(pTask, pReq, pRsp);
   tDeleteStreamDispatchReq(pReq);
 
   if (exec) {
@@ -232,12 +235,14 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
 
 int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) {
   ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED);
-  qDebug("task %d receive dispatch rsp, code: %x", pTask->id.taskId, code);
+  qDebug("s-task:%s receive dispatch rsp, code: %x", pTask->id.idStr, code);
 
   if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
     int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1);
     qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp);
-    if (leftRsp > 0) return 0;
+    if (leftRsp > 0) {
+      return 0;
+    }
   }
 
   int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus);
@@ -282,7 +287,7 @@ bool tInputQueueIsFull(const SStreamTask* pTask) {
   return taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUEU_CAPACITY;
 }
 
-int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) {
+int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) {
   int8_t type = pItem->type;
 
   if (type == STREAM_INPUT__DATA_SUBMIT) {
@@ -295,12 +300,12 @@ int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) {
     }
 
     int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1;
-    qDebug("s-task:%s submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->id.idStr,
-           pItem, pSubmitBlock, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen,
+    qDebug("s-task:%s submit enqueue %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->id.idStr,
+           pItem, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen,
            pSubmitBlock->submit.ver, total);
 
-    if (total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) {
-      qDebug("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY);
+    if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) {
+      qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY);
       streamDataSubmitDestroy(pSubmitBlock);
       return -1;
     }
@@ -309,8 +314,8 @@ int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) {
   } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE ||
              type == STREAM_INPUT__REF_DATA_BLOCK) {
     int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1;
-    if (total > 2) {
-      qDebug("stream task input queue is full, abort");
+    if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) {
+      qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY);
       return -1;
     }
 
@@ -327,7 +332,6 @@ int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) {
   }
 
 #if 0
-  // TODO: back pressure
   atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL);
 #endif
 
diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c
index 63d15f134d..ae616260f3 100644
--- a/source/libs/stream/src/streamData.c
+++ b/source/libs/stream/src/streamData.c
@@ -69,7 +69,6 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
 
 SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type) {
   SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0);
-
   if (pDataSubmit == NULL) {
     return NULL;
   }
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 4e491f906a..a9f6d29bf5 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -238,7 +238,8 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq*
   msg.pCont = buf;
   msg.msgType = TDMT_STREAM_TASK_CHECK;
 
-  qDebug("dispatch from task %d to task %d node %d: check msg", pTask->id.taskId, pReq->downstreamTaskId, nodeId);
+  qDebug("dispatch from s-task:%s to downstream s-task:%"PRIx64":%d node %d: check msg", pTask->id.idStr,
+      pReq->streamId, pReq->downstreamTaskId, nodeId);
 
   tmsgSendReq(pEpSet, &msg);
 
@@ -319,8 +320,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p
   msg.pCont = buf;
   msg.msgType = pTask->dispatchMsgType;
 
-  qDebug("dispatch from task %d to task %d node %d: data msg", pTask->id.taskId, pReq->taskId, vgId);
-
+  qDebug("dispatch from s-task:%s to taskId:%d vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId);
   tmsgSendReq(pEpSet, &msg);
 
   code = 0;
@@ -402,14 +402,15 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
         goto FAIL_FIXED_DISPATCH;
       }
     }
+
     int32_t vgId = pTask->fixedEpDispatcher.nodeId;
     SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet;
     int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId;
 
     req.taskId = downstreamTaskId;
 
-    qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->id.taskId, pTask->selfChildId,
-           downstreamTaskId, vgId);
+    qDebug("s-task:%s (child taskId:%d) dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr,
+           pTask->selfChildId, blockNum, downstreamTaskId, vgId);
 
     if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) {
       goto FAIL_FIXED_DISPATCH;
@@ -494,6 +495,8 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
 
 int32_t streamDispatch(SStreamTask* pTask) {
   ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH);
+  qDebug("s-task:%s try to dispatch intermediate result block to downstream, numofBlocks in outputQ:%d", pTask->id.idStr,
+         taosQueueItemSize(pTask->outputQueue->queue));
 
   int8_t old =
       atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT);
@@ -503,13 +506,12 @@ int32_t streamDispatch(SStreamTask* pTask) {
 
   SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue);
   if (pBlock == NULL) {
-    qDebug("stream stop dispatching since no output: task %d", pTask->id.taskId);
+    qDebug("s-task:%s stream stop dispatching since no output in output queue", pTask->id.idStr);
     atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
     return 0;
   }
-  ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
 
-  qDebug("stream dispatching: task %d", pTask->id.taskId);
+  ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
 
   int32_t code = 0;
   if (streamDispatchAllBlocks(pTask, pBlock) < 0) {
@@ -518,6 +520,7 @@ int32_t streamDispatch(SStreamTask* pTask) {
     atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
     goto FREE;
   }
+
 FREE:
   taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
   taosFreeQitem(pBlock);
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index db9be593c0..3d896c08ac 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -40,9 +40,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
   } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
     ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
     const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data;
-    qDebug("s-task:%s set submit blocks as input %p %p %d ver:%" PRId64, pTask->id.idStr, pSubmit, pSubmit->submit.msgStr,
-           pSubmit->submit.msgLen, pSubmit->submit.ver);
     qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
+    qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, pSubmit->submit.msgStr,
+           pSubmit->submit.msgLen, pSubmit->submit.ver);
   } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
     const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data;
 
@@ -241,7 +241,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
     while (1) {
       SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
       if (qItem == NULL) {
-        qDebug("s-task:%s stream task exec over, queue empty", pTask->id.idStr);
+//        qDebug("s-task:%s extract data from input queue, queue is empty, abort", pTask->id.idStr);
         break;
       }
 
@@ -280,12 +280,13 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 
     if (pTask->taskLevel == TASK_LEVEL__SINK) {
       ASSERT(((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_BLOCK);
+      qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize);
       streamTaskOutput(pTask, pInput);
       continue;
     }
 
     SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
-    qDebug("s-task:%s exec begin, msg batch: %d", pTask->id.idStr, batchSize);
+    qDebug("s-task:%s exec begin, numOfBlocks:%d", pTask->id.idStr, batchSize);
 
     streamTaskExecImpl(pTask, pInput, pRes);
 
@@ -293,13 +294,21 @@ int32_t streamExecForAll(SStreamTask* pTask) {
     int64_t dataVer = 0;
     qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId);
     if (dataVer > pTask->chkInfo.version) {    // save it since the checkpoint is updated
-      qDebug("s-task:%s exec end, checkpoint ver from %"PRId64" to %"PRId64, pTask->id.idStr, pTask->chkInfo.version, dataVer);
-      pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId};
-      streamMetaSaveTask(pTask->pMeta, pTask);
+      qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64
+             ", checkPoint id:%" PRId64 " -> %" PRId64,
+             pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId);
 
+      pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId};
+
+      taosWLockLatch(&pTask->pMeta->lock);
+      streamMetaSaveTask(pTask->pMeta, pTask);
       if (streamMetaCommit(pTask->pMeta) < 0) {
-        qError("failed to commit stream meta, since %s", terrstr());
+        taosWUnLockLatch(&pTask->pMeta->lock);
+        qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr());
         return -1;
+      } else {
+        taosWUnLockLatch(&pTask->pMeta->lock);
+        qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr);
       }
     } else {
       qDebug("s-task:%s exec end", pTask->id.idStr);
@@ -354,6 +363,7 @@ int32_t streamTryExec(SStreamTask* pTask) {
 
     // todo the task should be commit here
     atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
+    qDebug("s-task:%s exec completed", pTask->id.idStr);
 
     if (!taosQueueEmpty(pTask->inputQueue->queue)) {
       streamSchedExec(pTask);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 2e9bb4d762..4b423cc432 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -51,8 +51,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     goto _err;
   }
 
-  pMeta->pRestoreTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK);
-  if (pMeta->pRestoreTasks == NULL) {
+  pMeta->pWalReadTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK);
+  if (pMeta->pWalReadTasks == NULL) {
     goto _err;
   }
 
@@ -60,15 +60,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     goto _err;
   }
 
+  pMeta->vgId = vgId;
   pMeta->ahandle = ahandle;
   pMeta->expandFunc = expandFunc;
-
+  taosInitRWLatch(&pMeta->lock);
   return pMeta;
 
 _err:
   taosMemoryFree(pMeta->path);
   if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks);
-  if (pMeta->pRestoreTasks) taosHashCleanup(pMeta->pRestoreTasks);
+  if (pMeta->pWalReadTasks) taosHashCleanup(pMeta->pWalReadTasks);
   if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb);
   if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb);
   if (pMeta->db) tdbClose(pMeta->db);
@@ -83,20 +84,29 @@ void streamMetaClose(SStreamMeta* pMeta) {
   tdbClose(pMeta->db);
 
   void* pIter = NULL;
+  while(pMeta->walScan) {
+    qDebug("wait stream daemon quit");
+    taosMsleep(100);
+  }
+
   while (1) {
     pIter = taosHashIterate(pMeta->pTasks, pIter);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
+
     SStreamTask* pTask = *(SStreamTask**)pIter;
     if (pTask->timer) {
       taosTmrStop(pTask->timer);
       pTask->timer = NULL;
     }
+
     tFreeStreamTask(pTask);
     /*streamMetaReleaseTask(pMeta, pTask);*/
   }
 
   taosHashCleanup(pMeta->pTasks);
-  taosHashCleanup(pMeta->pRestoreTasks);
+  taosHashCleanup(pMeta->pWalReadTasks);
   taosMemoryFree(pMeta->path);
   taosMemoryFree(pMeta);
 }
@@ -164,8 +174,8 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   return 0;
 }
 
-#if 1
-int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
+// add to the ready tasks hash map, not the restored tasks hash map
+int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
   if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) {
     return -1;
   }
@@ -174,10 +184,16 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
     return -1;
   }
 
-  taosHashPut(pMeta->pRestoreTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES);
+  pTask->status.taskStatus = STREAM_STATUS__NORMAL;
+  taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES);
   return 0;
 }
-#endif
+
+int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) {
+  int32_t numOfReady = taosHashGetSize(pMeta->pTasks);
+  int32_t numOfRestoring = taosHashGetSize(pMeta->pWalReadTasks);
+  return numOfReady + numOfRestoring;
+}
 
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
   taosRLockLatch(&pMeta->lock);
@@ -206,9 +222,9 @@ SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId) {
   taosRLockLatch(&pMeta->lock);
 
   SStreamTask* pTask = NULL;
-  int32_t numOfRestored = taosHashGetSize(pMeta->pRestoreTasks);
+  int32_t      numOfRestored = taosHashGetSize(pMeta->pWalReadTasks);
   if (numOfRestored > 0) {
-    SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pRestoreTasks, &taskId, sizeof(int32_t));
+    SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pWalReadTasks, &taskId, sizeof(taskId));
     if (p != NULL) {
       pTask = *p;
       if (pTask != NULL && (atomic_load_8(&(pTask->status.taskStatus)) != TASK_STATUS__DROPPING)) {
@@ -217,15 +233,15 @@ SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId) {
         return pTask;
       }
     }
-  } else {
-    SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
-    if (p != NULL) {
-      pTask = *p;
-      if (pTask != NULL && atomic_load_8(&(pTask->status.taskStatus)) != TASK_STATUS__DROPPING) {
-        atomic_add_fetch_32(&pTask->refCnt, 1);
-        taosRUnLockLatch(&pMeta->lock);
-        return pTask;
-      }
+  }
+
+  SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
+  if (p != NULL) {
+    pTask = *p;
+    if (pTask != NULL && atomic_load_8(&(pTask->status.taskStatus)) != TASK_STATUS__DROPPING) {
+      atomic_add_fetch_32(&pTask->refCnt, 1);
+      taosRUnLockLatch(&pMeta->lock);
+      return pTask;
     }
   }
 
@@ -261,9 +277,12 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) {
 
 int32_t streamMetaCommit(SStreamMeta* pMeta) {
   if (tdbCommit(pMeta->db, pMeta->txn) < 0) {
+    ASSERT(0);
     return -1;
   }
+
   if (tdbPostCommit(pMeta->db, pMeta->txn) < 0) {
+    ASSERT(0);
     return -1;
   }
 
@@ -319,7 +338,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
       return -1;
     }
 
-    if (taosHashPut(pMeta->pRestoreTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
+    if (taosHashPut(pMeta->pWalReadTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
       tdbFree(pKey);
       tdbFree(pVal);
       tdbTbcClose(pCur);
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 834c022a9a..7d2d7a666f 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -186,7 +186,8 @@ void tFreeStreamTask(SStreamTask* pTask) {
     pTask->exec.pExecutor = NULL;
   }
 
-  if (pTask->exec.pTqReader != NULL) {
+  if (pTask->exec.pTqReader != NULL && pTask->freeFp != NULL) {
+    pTask->freeFp(pTask->exec.pTqReader);
     pTask->exec.pTqReader = NULL;
   }
 
@@ -206,5 +207,9 @@ void tFreeStreamTask(SStreamTask* pTask) {
     streamStateClose(pTask->pState);
   }
 
+  if (pTask->id.idStr != NULL)  {
+    taosMemoryFree((void*)pTask->id.idStr);
+  }
+
   taosMemoryFree(pTask);
 }
diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c
index d57104dd78..a49ff0cd5b 100644
--- a/source/util/src/tworker.c
+++ b/source/util/src/tworker.c
@@ -248,7 +248,8 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem
     }
 
     taosThreadAttrDestroy(&thAttr);
-    uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, (int32_t)taosArrayGetSize(pool->workers));
+    int32_t numOfThreads = taosArrayGetSize(pool->workers);
+    uInfo("worker:%s:%d is launched, total:%d, expect:%d", pool->name, worker->id, numOfThreads, dstWorkerNum);
 
     curWorkerNum++;
   }
diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim
index e69875d69f..15ca6bf7c9 100644
--- a/tests/script/tsim/stream/basic1.sim
+++ b/tests/script/tsim/stream/basic1.sim
@@ -37,7 +37,7 @@ if $loop_count == 20 then
 endi
 
 if $rows != 4 then
-  print =====rows=$rows
+  print =====rows=$rows, expect 4
   goto loop0
 endi
 
@@ -53,7 +53,7 @@ if $data02 != 2 then
 endi
 
 if $data03 != 5 then
-  print =====data03=$data03
+  print =====data03=$data03,  expect:5
   goto loop0
 endi
 

From 4d83118ff756b588c4f5744195020c2612a09b4b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 13 Apr 2023 23:49:14 +0800
Subject: [PATCH 34/65] fix(stream): fix memory leak.

---
 source/dnode/vnode/src/tq/tq.c      | 1 +
 source/libs/stream/CMakeLists.txt   | 2 +-
 source/libs/stream/src/streamTask.c | 5 +++++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 2b911befcc..59a78efe53 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -635,6 +635,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
     pTask->freeFp = (_free_reader_fn_t)tqCloseReader;
     SArray* pList = qGetQueriedTableListInfo(pTask->exec.pExecutor);
     tqReaderAddTbUidList(pTask->exec.pTqReader, pList);
+    taosArrayDestroy(pList);
   }
 
   streamSetupTrigger(pTask);
diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt
index ceddf4f215..790547bb61 100644
--- a/source/libs/stream/CMakeLists.txt
+++ b/source/libs/stream/CMakeLists.txt
@@ -9,7 +9,7 @@ target_include_directories(
 target_link_libraries(
     stream
     PUBLIC tdb
-    PRIVATE os util transport qcom executor
+    PRIVATE os util transport qcom executor wal
 )
 
 if(${BUILD_TEST})
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 7d2d7a666f..4783997276 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -15,6 +15,7 @@
 
 #include "executor.h"
 #include "tstream.h"
+#include "wal.h"
 
 SStreamTask* tNewStreamTask(int64_t streamId) {
   SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
@@ -191,6 +192,10 @@ void tFreeStreamTask(SStreamTask* pTask) {
     pTask->exec.pTqReader = NULL;
   }
 
+  if (pTask->exec.pWalReader != NULL) {
+    walCloseReader(pTask->exec.pWalReader);
+  }
+
   taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree);
   if (pTask->outputType == TASK_OUTPUT__TABLE) {
     tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper);

From 99587db7a10b7a6d5aeb28c615c4becaf6e90584 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 14 Apr 2023 07:58:25 +0800
Subject: [PATCH 35/65] fix: udf handle expired after 10s

---
 source/libs/function/src/tudf.c | 30 ++++++++++++++++--------------
 source/libs/function/src/udfd.c |  2 +-
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 53b2cf4aca..f13235f24b 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -960,7 +960,7 @@ int32_t udfcOpen();
 int32_t udfcClose();
 
 int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle);
-void    releaseUdfFuncHandle(char *udfName);
+void    releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle);
 int32_t cleanUpUdfs();
 
 bool    udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
@@ -992,11 +992,12 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
         uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
         return 0;
       } else {
-        fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName,
+        fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName,
                foundStub->refCount, foundStub->createTime);
         taosArrayRemove(gUdfcProxy.udfStubs, stubIndex);
       }
     } else {
+      fnInfo("udf handle expired for %s, will setup udf. move it to expired list", udfName);
       taosArrayRemove(gUdfcProxy.udfStubs, stubIndex);
       taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub);
       taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub);
@@ -1020,7 +1021,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
   return code;
 }
 
-void releaseUdfFuncHandle(char *udfName) {
+void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) {
   uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
   SUdfcFuncStub key = {0};
   strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
@@ -1030,10 +1031,10 @@ void releaseUdfFuncHandle(char *udfName) {
     uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
     return;
   }
-  if (foundStub != NULL && foundStub->refCount > 0) {
+  if (foundStub != NULL && foundStub->handle == handle && foundStub->refCount > 0) {
     --foundStub->refCount;
   }
-  if (expiredStub != NULL && expiredStub->refCount > 0) {
+  if (expiredStub != NULL && expiredStub->handle == handle && expiredStub->refCount > 0) {
     --expiredStub->refCount;
   }
   uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
@@ -1046,7 +1047,8 @@ int32_t cleanUpUdfs() {
   }
 
   uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
-  if (gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) {
+  if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) &&
+      (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) {
     uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
     return TSDB_CODE_SUCCESS;
   }
@@ -1092,7 +1094,7 @@ int32_t cleanUpUdfs() {
     }
     ++i;
   }
-  taosArrayDestroy(gUdfcProxy.udfStubs);
+  taosArrayDestroy(gUdfcProxy.expiredUdfStubs);
   gUdfcProxy.expiredUdfStubs = expiredUdfStubs;
   uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
   return 0;
@@ -1109,7 +1111,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols,
   code = doCallUdfScalarFunc(handle, input, numOfCols, output);
   if (code != TSDB_CODE_SUCCESS) {
     fnError("udfc scalar function execution failure");
-    releaseUdfFuncHandle(udfName);
+    releaseUdfFuncHandle(udfName, handle);
     return code;
   }
 
@@ -1123,7 +1125,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols,
       code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE;
     }
   }
-  releaseUdfFuncHandle(udfName);
+  releaseUdfFuncHandle(udfName, handle);
   return code;
 }
 
@@ -1156,7 +1158,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult
   SUdfInterBuf buf = {0};
   if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) {
     fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode);
-    releaseUdfFuncHandle(pCtx->udfName);
+    releaseUdfFuncHandle(pCtx->udfName, handle);
     return false;
   }
   if (buf.bufLen <= session->bufSize) {
@@ -1165,10 +1167,10 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult
     udfRes->interResNum = buf.numOfResult;
   } else {
     fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize);
-    releaseUdfFuncHandle(pCtx->udfName);
+    releaseUdfFuncHandle(pCtx->udfName, handle);
     return false;
   }
-  releaseUdfFuncHandle(pCtx->udfName);
+  releaseUdfFuncHandle(pCtx->udfName, handle);
   freeUdfInterBuf(&buf);
   return true;
 }
@@ -1225,7 +1227,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
   taosArrayDestroy(pTempBlock->pDataBlock);
   taosMemoryFree(pTempBlock);
 
-  releaseUdfFuncHandle(pCtx->udfName);
+  releaseUdfFuncHandle(pCtx->udfName, handle);
   freeUdfInterBuf(&newState);
   return udfCode;
 }
@@ -1270,7 +1272,7 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock) {
   freeUdfInterBuf(&resultBuf);
 
   int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
-  releaseUdfFuncHandle(pCtx->udfName);
+  releaseUdfFuncHandle(pCtx->udfName, handle);
   return udfCallCode == 0 ? numOfResults : udfCallCode;
 }
 
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index aa72309c62..61c6f92e2f 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -591,7 +591,7 @@ SUdf *udfdNewUdf(const char *udfName) {
 SUdf *udfdGetOrCreateUdf(const char *udfName) {
   uv_mutex_lock(&global.udfsMutex);
   SUdf  **pUdfHash = taosHashGet(global.udfsHash, udfName, strlen(udfName));
-  int64_t currTime = taosGetTimestampSec();
+  int64_t currTime = taosGetTimestampMs();
   bool    expired = false;
   if (pUdfHash) {
     expired = currTime - (*pUdfHash)->lastFetchTime > 10 * 1000;  // 10s

From e985f15cf07cde40b3271bd1cc8e5f00cd295905 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 14 Apr 2023 09:36:36 +0800
Subject: [PATCH 36/65] fix:memset nullBitmap of SSDataBlock to 0 in udf

---
 include/libs/function/taosudf.h | 2 ++
 source/common/src/tdatablock.c  | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h
index 1b1339340b..47f4956206 100644
--- a/include/libs/function/taosudf.h
+++ b/include/libs/function/taosudf.h
@@ -166,6 +166,8 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
+    uint32_t extend = BitmapLen(allocCapacity) - BitmapLen(data->rowsAlloc);
+    memset(tmp + BitmapLen(data->rowsAlloc), 0, extend);
     data->fixLenCol.nullBitmap = tmp;
     data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity);
     if (meta->type == TSDB_DATA_TYPE_NULL) {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 986a46036b..2cdcfecbdd 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -314,7 +314,8 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int
         if (btmp == NULL) {
           return TSDB_CODE_OUT_OF_MEMORY;
         }
-
+        uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1);
+        memset(btmp + BitmapLen(numOfRow1), 0, extend);
         pColumnInfoData->nullbitmap = btmp;
       }
 

From eb0e1f848a2939fa8ea8e7bfb2263c5b1db409ab Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 14 Apr 2023 12:00:09 +0800
Subject: [PATCH 37/65] fix(stream): remove unused tqreader, do some internal
 refactor, set the meta value for streamtask.

---
 include/libs/executor/executor.h           |  2 +-
 include/libs/stream/tstream.h              |  3 --
 source/dnode/mgmt/mgmt_snode/src/smInt.c   |  1 +
 source/dnode/mnode/impl/src/mndScheduler.c |  1 +
 source/dnode/snode/src/snode.c             |  3 ++
 source/dnode/vnode/src/sma/smaRollup.c     |  2 +-
 source/dnode/vnode/src/tq/tq.c             | 13 +-------
 source/dnode/vnode/src/tq/tqRead.c         | 11 ++-----
 source/dnode/vnode/src/tq/tqUtil.c         | 31 ++-----------------
 source/libs/executor/src/executor.c        |  7 +----
 source/libs/stream/src/streamMeta.c        | 36 ++++------------------
 source/libs/stream/src/streamRecover.c     |  9 ++----
 source/libs/stream/src/streamTask.c        |  5 ---
 13 files changed, 22 insertions(+), 102 deletions(-)

diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 6f2a6126b3..34372dc2ff 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -123,7 +123,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
  * @param isAdd
  * @return
  */
-int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd, SArray* pList);
+int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd);
 
 /**
  * Create the exec task object according to task json
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index eea8868b8c..4b4ef55485 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -221,7 +221,6 @@ SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit);
 typedef struct {
   char*              qmsg;
   void*              pExecutor;   // not applicable to encoder and decoder
-  struct STqReader*  pTqReader;   // not applicable to encoder and decoder
   struct SWalReader* pWalReader;  // not applicable to encoder and decoder
 } STaskExec;
 
@@ -333,7 +332,6 @@ struct SStreamTask {
   int64_t             checkpointingId;
   int32_t             checkpointAlignCnt;
   struct SStreamMeta* pMeta;
-  _free_reader_fn_t   freeFp;
 };
 
 // meta
@@ -343,7 +341,6 @@ typedef struct SStreamMeta {
     TTB*         pTaskDb;
     TTB*         pCheckpointDb;
     SHashObj*    pTasks;
-    SHashObj*    pWalReadTasks;
     void*        ahandle;
     TXN*         txn;
     FTaskExpand* expandFunc;
diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c
index 28097311ac..e222349767 100644
--- a/source/dnode/mgmt/mgmt_snode/src/smInt.c
+++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c
@@ -55,6 +55,7 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
     smClose(pMgmt);
     return -1;
   }
+
   tmsgReportStartup("snode-impl", "initialized");
 
   if (smStartWorker(pMgmt) != 0) {
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 36521fd778..734f624be0 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -356,6 +356,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
         qDestroyQueryPlan(pPlan);
         return -1;
       }
+
       pInnerTask->fillHistory = pStream->fillHistory;
       mndAddTaskToTaskSet(taskInnerLevel, pInnerTask);
 
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 4235548e48..cefc4fa63e 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -32,6 +32,7 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) {
     tDecoderClear(&decoder);
     goto FAIL;
   }
+
   tDecoderClear(&decoder);
 
   int32_t taskId = req.taskId;
@@ -77,6 +78,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
   pTask->pMsgCb = &pSnode->msgCb;
   pTask->chkInfo.version = ver;
+  pTask->pMeta = pSnode->pMeta;
 
   pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1);
   if (pTask->pState == NULL) {
@@ -137,6 +139,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
   if (pTask == NULL) {
     return -1;
   }
+
   SDecoder decoder;
   tDecoderInit(&decoder, (uint8_t *)msg, msgLen);
   code = tDecodeStreamTask(&decoder, pTask);
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 8aeb705d90..ce987ca88e 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -168,7 +168,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids,
 
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pRSmaInfo->taskInfo[i]) {
-      if ((terrno = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd, NULL)) < 0) {
+      if ((terrno = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) {
         tdReleaseRSmaInfo(pSma, pRSmaInfo);
         smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i,
                  terrstr());
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 5f3facf476..5d4fdbe5d8 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -567,6 +567,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
   pTask->pMsgCb = &pTq->pVnode->msgCb;
   pTask->pMeta = pTq->pStreamMeta;
+  pTask->chkInfo.version = ver;
 
   // expand executor
   if (pTask->fillHistory) {
@@ -628,18 +629,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   }
 
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
-    pTask->exec.pTqReader = tqOpenReader(pTq->pVnode);
-    if (pTask->exec.pTqReader == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-
     pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
-
-    pTask->freeFp = (_free_reader_fn_t)tqCloseReader;
-    SArray* pList = qGetQueriedTableListInfo(pTask->exec.pExecutor);
-    tqReaderAddTbUidList(pTask->exec.pTqReader, pList);
-    taosArrayDestroy(pList);
   }
 
   streamSetupTrigger(pTask);
@@ -1141,7 +1131,6 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
     }
 
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
-
     tqStartStreamTasks(pTq);
     return 0;
   } else {
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 69624f4d10..25ab7209d2 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -973,7 +973,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
 
     STqHandle* pTqHandle = (STqHandle*)pIter;
     if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-      int32_t code = qUpdateTableListForStreamScanner(pTqHandle->execHandle.task, tbUidList, isAdd, NULL);
+      int32_t code = qUpdateTableListForStreamScanner(pTqHandle->execHandle.task, tbUidList, isAdd);
       if (code != 0) {
         tqError("update qualified table error for %s", pTqHandle->subKey);
         continue;
@@ -1031,18 +1031,11 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
 
     SStreamTask* pTask = *(SStreamTask**)pIter;
     if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
-      SArray* pList = NULL;
-      int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd, pList);
+      int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd);
       if (code != 0) {
         tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr);
         continue;
       }
-
-      if (isAdd) { // only add qualified tables
-        tqReaderAddTbUidList(pTask->exec.pTqReader, pList);
-      } else {
-        tqReaderRemoveTbUidList(pTask->exec.pTqReader, tbUidList);
-      }
     }
   }
 
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 791bfbe6df..4c37e1052f 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -49,32 +49,6 @@ int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueI
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset) {
-  SStreamDataBlock* pBlocks = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
-  if (pBlocks == NULL) { // failed, do nothing
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pRet->data.info.type = STREAM_NORMAL;
-  pBlocks->type = STREAM_INPUT__DATA_BLOCK;
-  pBlocks->sourceVer = pOffset->val.version;
-  pBlocks->blocks = taosArrayInit(0, sizeof(SSDataBlock));
-  taosArrayPush(pBlocks->blocks, &pRet->data);
-
-//          int64_t* ts = (int64_t*)(((SColumnInfoData*)ret.data.pDataBlock->pData)->pData);
-//          tqDebug("-----------%ld\n", ts[0]);
-
-  int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pBlocks, pBlocks->sourceVer);
-  if (code == TSDB_CODE_SUCCESS) {
-    pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pTqReader->pWalReader);
-    tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
-            pOffset->val.version);
-  }
-
-  return 0;
-}
-
 void initOffsetForAllRestoreTasks(STQ* pTq) {
   void* pIter = NULL;
 
@@ -90,8 +64,7 @@ void initOffsetForAllRestoreTasks(STQ* pTq) {
     }
 
     if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
-              pTask->status.taskStatus);
+      tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus);
       continue;
     }
 
@@ -120,7 +93,7 @@ void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
     }
 
     if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
+      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
               pTask->status.taskStatus);
       continue;
     }
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 255949a588..2bc91f0cb3 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -370,7 +370,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
   return qa;
 }
 
-int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd, SArray* pList) {
+int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
   const char*    id = GET_TASKID(pTaskInfo);
   int32_t        code = 0;
@@ -386,11 +386,6 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
   if (isAdd) {  // add new table id
     SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
     int32_t numOfQualifiedTables = taosArrayGetSize(qa);
-
-    if (pList != NULL) {
-      taosArrayAddAll(pList, qa);
-    }
-
     qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id);
     code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
     if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 0008c8dd8c..c45c700375 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -57,11 +57,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     goto _err;
   }
 
-  pMeta->pWalReadTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK);
-  if (pMeta->pWalReadTasks == NULL) {
-    goto _err;
-  }
-
   if (streamMetaBegin(pMeta) < 0) {
     goto _err;
   }
@@ -75,7 +70,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
 _err:
   taosMemoryFree(pMeta->path);
   if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks);
-  if (pMeta->pWalReadTasks) taosHashCleanup(pMeta->pWalReadTasks);
   if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb);
   if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb);
   if (pMeta->db) tdbClose(pMeta->db);
@@ -112,7 +106,6 @@ void streamMetaClose(SStreamMeta* pMeta) {
   }
 
   taosHashCleanup(pMeta->pTasks);
-  taosHashCleanup(pMeta->pWalReadTasks);
   taosMemoryFree(pMeta->path);
   taosMemoryFree(pMeta);
 }
@@ -196,9 +189,7 @@ int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask*
 }
 
 int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) {
-  int32_t numOfReady = taosHashGetSize(pMeta->pTasks);
-  int32_t numOfRestoring = taosHashGetSize(pMeta->pWalReadTasks);
-  return numOfReady + numOfRestoring;
+  return (int32_t) taosHashGetSize(pMeta->pTasks);
 }
 
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
@@ -225,34 +216,19 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
 }
 
 SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId) {
-  taosRLockLatch(&pMeta->lock);
-
   SStreamTask* pTask = NULL;
-  int32_t      numOfRestored = taosHashGetSize(pMeta->pWalReadTasks);
-  if (numOfRestored > 0) {
-    SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pWalReadTasks, &taskId, sizeof(taskId));
-    if (p != NULL) {
-      pTask = *p;
-      if (pTask != NULL && (atomic_load_8(&(pTask->status.taskStatus)) != TASK_STATUS__DROPPING)) {
-        atomic_add_fetch_32(&pTask->refCnt, 1);
-        taosRUnLockLatch(&pMeta->lock);
-        return pTask;
-      }
-    }
-  }
+  taosRLockLatch(&pMeta->lock);
 
   SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
   if (p != NULL) {
-    pTask = *p;
-    if (pTask != NULL && atomic_load_8(&(pTask->status.taskStatus)) != TASK_STATUS__DROPPING) {
+    if ((*p) != NULL && atomic_load_8(&((*p)->status.taskStatus)) != TASK_STATUS__DROPPING) {
+      pTask = *p;
       atomic_add_fetch_32(&pTask->refCnt, 1);
-      taosRUnLockLatch(&pMeta->lock);
-      return pTask;
     }
   }
 
   taosRUnLockLatch(&pMeta->lock);
-  return NULL;
+  return pTask;
 }
 
 void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
@@ -344,7 +320,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
       return -1;
     }
 
-    if (taosHashPut(pMeta->pWalReadTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
+    if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
       tdbFree(pKey);
       tdbFree(pVal);
       tdbTbcClose(pCur);
diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c
index 9962cdfcc0..6c24e69832 100644
--- a/source/libs/stream/src/streamRecover.c
+++ b/source/libs/stream/src/streamRecover.c
@@ -17,6 +17,7 @@
 
 int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
   qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId);
+
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
     atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE);
     streamSetParamForRecover(pTask);
@@ -33,12 +34,7 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
 
     memcpy(serializedReq, &req, len);
 
-    SRpcMsg rpcMsg = {
-        .contLen = len,
-        .pCont = serializedReq,
-        .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE,
-    };
-
+    SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE };
     if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) {
       /*ASSERT(0);*/
     }
@@ -61,6 +57,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
       .upstreamNodeId = pTask->nodeId,
       .childId = pTask->selfChildId,
   };
+
   // serialize
   if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
     req.reqId = tGenIdPI64();
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 4783997276..67c60008fd 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -187,11 +187,6 @@ void tFreeStreamTask(SStreamTask* pTask) {
     pTask->exec.pExecutor = NULL;
   }
 
-  if (pTask->exec.pTqReader != NULL && pTask->freeFp != NULL) {
-    pTask->freeFp(pTask->exec.pTqReader);
-    pTask->exec.pTqReader = NULL;
-  }
-
   if (pTask->exec.pWalReader != NULL) {
     walCloseReader(pTask->exec.pWalReader);
   }

From eef0c845ed08d4c9e9b1f591214cd8514c42ff6c Mon Sep 17 00:00:00 2001
From: xiaolei li <85657333+xleili@users.noreply.github.com>
Date: Fri, 14 Apr 2023 13:28:55 +0800
Subject: [PATCH 38/65] enchance: increate testpackage.sh timeout seconds
 (#20920)

* ench: increate testpackage.sh timeout seconds

* Update debRpmAutoInstall.sh

extend timeout to 30
---
 packaging/debRpmAutoInstall.sh | 2 +-
 packaging/testpackage.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/debRpmAutoInstall.sh b/packaging/debRpmAutoInstall.sh
index 2fe18fd7a9..8fadffe4c6 100755
--- a/packaging/debRpmAutoInstall.sh
+++ b/packaging/debRpmAutoInstall.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/expect 
 set packageName [lindex $argv 0]
 set packageSuffix [lindex $argv 1]
-set timeout 3 
+set timeout 30 
 if { ${packageSuffix} == "deb" } {
     spawn  dpkg -i ${packageName}
 } elseif { ${packageSuffix} == "rpm"} {
diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index 97226a86b5..081383f89b 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -246,7 +246,7 @@ if [ ! -f  debRpmAutoInstall.sh  ];then
     echo '#!/usr/bin/expect ' >  debRpmAutoInstall.sh
     echo 'set packageName [lindex $argv 0]' >>  debRpmAutoInstall.sh
     echo 'set packageSuffix [lindex $argv 1]' >>  debRpmAutoInstall.sh
-    echo 'set timeout 3 ' >>  debRpmAutoInstall.sh
+    echo 'set timeout 30 ' >>  debRpmAutoInstall.sh
     echo 'if { ${packageSuffix} == "deb" } {' >>  debRpmAutoInstall.sh
     echo '    spawn  dpkg -i ${packageName} '  >>  debRpmAutoInstall.sh
     echo '} elseif { ${packageSuffix} == "rpm"} {' >>  debRpmAutoInstall.sh

From e0cb8aa534f70a760df4f7272ab25b4ffa13af20 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 14 Apr 2023 17:43:37 +0800
Subject: [PATCH 39/65] fix(stream): don't the initial task status and do some
 internal refactor.

---
 include/libs/stream/tstream.h          |  2 --
 source/dnode/vnode/src/tq/tq.c         | 35 +++++++++++++-------------
 source/libs/stream/src/streamMeta.c    | 17 -------------
 source/libs/stream/src/streamRecover.c | 17 ++++++++++---
 4 files changed, 31 insertions(+), 40 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 4b4ef55485..103f807191 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -31,7 +31,6 @@ extern "C" {
 #ifndef _STREAM_H_
 #define _STREAM_H_
 
-typedef void  (*_free_reader_fn_t)(void*);
 typedef struct SStreamTask SStreamTask;
 
 enum {
@@ -574,7 +573,6 @@ int32_t      streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamT
 int32_t      streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen);
 int32_t      streamMetaGetNumOfTasks(const SStreamMeta* pMeta);
 
-SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 void         streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 5d4fdbe5d8..1230a352d9 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -659,17 +659,22 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
   };
 
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
-  if (pTask && atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL) {
-    rsp.status = 1;
+  if (pTask) {
+    rsp.status = (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL) ? 1 : 0;
+    streamMetaReleaseTask(pTq->pStreamMeta, pTask);
+
+    tqDebug("tq recv task check req(reqId:0x%" PRIx64
+            ") %d at node %d task status:%d, check req from task %d at node %d, rsp status %d",
+            rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus, rsp.upstreamTaskId,
+            rsp.upstreamNodeId, rsp.status);
   } else {
     rsp.status = 0;
+    tqDebug("tq recv task check(taskId:%d not built yet) req(reqId:0x%" PRIx64
+            ") %d at node %d, check req from task %d at node %d, rsp status %d",
+            taskId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId,
+            rsp.status);
   }
 
-  if (pTask) streamMetaReleaseTask(pTq->pStreamMeta, pTask);
-
-  tqDebug("tq recv task check req(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d",
-          rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
-
   SEncoder encoder;
   int32_t  code;
   int32_t  len;
@@ -687,13 +692,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
   tEncodeSStreamTaskCheckRsp(&encoder, &rsp);
   tEncoderClear(&encoder);
 
-  SRpcMsg rspMsg = {
-      .code = 0,
-      .pCont = buf,
-      .contLen = sizeof(SMsgHead) + len,
-      .info = pMsg->info,
-  };
-
+  SRpcMsg rspMsg = { .code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info };
   tmsgSendRsp(&rspMsg);
   return 0;
 }
@@ -709,8 +708,8 @@ int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32
     tDecoderClear(&decoder);
     return -1;
   }
-  tDecoderClear(&decoder);
 
+  tDecoderClear(&decoder);
   tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d",
           rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
 
@@ -764,8 +763,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
     streamTaskCheckDownstream(pTask, sversion);
   }
 
-  tqDebug("vgId:%d s-task:%s is deployed from mnd, status:%d, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr,
-          pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta));
+  tqDebug("vgId:%d s-task:%s is deployed and add meta from mnd, status:%d, total:%d", TD_VID(pTq->pVnode),
+          pTask->id.idStr, pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta));
   return 0;
 }
 
@@ -1117,7 +1116,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
     return 0;
   }
 
-  SStreamTask* pTask = streamMetaAcquireTaskEx(pTq->pStreamMeta, taskId);
+  SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
   if (pTask != NULL) {
     if (pTask->status.taskStatus == TASK_STATUS__NORMAL) {
       tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index c45c700375..fecc01f295 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -183,7 +183,6 @@ int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask*
     return -1;
   }
 
-  pTask->status.taskStatus = STREAM_STATUS__NORMAL;
   taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES);
   return 0;
 }
@@ -215,22 +214,6 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   }
 }
 
-SStreamTask* streamMetaAcquireTaskEx(SStreamMeta* pMeta, int32_t taskId) {
-  SStreamTask* pTask = NULL;
-  taosRLockLatch(&pMeta->lock);
-
-  SStreamTask** p = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
-  if (p != NULL) {
-    if ((*p) != NULL && atomic_load_8(&((*p)->status.taskStatus)) != TASK_STATUS__DROPPING) {
-      pTask = *p;
-      atomic_add_fetch_32(&pTask->refCnt, 1);
-    }
-  }
-
-  taosRUnLockLatch(&pMeta->lock);
-  return pTask;
-}
-
 void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
   if (ppTask) {
diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c
index 6c24e69832..03afc0692d 100644
--- a/source/libs/stream/src/streamRecover.c
+++ b/source/libs/stream/src/streamRecover.c
@@ -46,6 +46,7 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
   } else if (pTask->taskLevel == TASK_LEVEL__SINK) {
     atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
   }
+
   return 0;
 }
 
@@ -125,6 +126,7 @@ int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq*
 int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) {
   qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId,
          pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status);
+
   if (pRsp->status == 1) {
     if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
       bool found = false;
@@ -135,7 +137,11 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp*
           break;
         }
       }
-      if (!found) return -1;
+
+      if (!found) {
+        return -1;
+      }
+
       int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1);
       ASSERT(left >= 0);
       if (left == 0) {
@@ -144,7 +150,10 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp*
         streamTaskLaunchRecover(pTask, version);
       }
     } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
-      if (pRsp->reqId != pTask->checkReqId) return -1;
+      if (pRsp->reqId != pTask->checkReqId) {
+        return -1;
+      }
+
       streamTaskLaunchRecover(pTask, version);
     } else {
       ASSERT(0);
@@ -164,6 +173,7 @@ int32_t streamRestoreParam(SStreamTask* pTask) {
   void* exec = pTask->exec.pExecutor;
   return qStreamRestoreParam(exec);
 }
+
 int32_t streamSetStatusNormal(SStreamTask* pTask) {
   atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
   return 0;
@@ -224,8 +234,8 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) {
 
 // agg
 int32_t streamAggRecoverPrepare(SStreamTask* pTask) {
-  void* exec = pTask->exec.pExecutor;
   pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo);
+  qDebug("s-task:%s wait for %d upstreams", pTask->id.idStr, pTask->recoverWaitingUpstream);
   return 0;
 }
 
@@ -244,6 +254,7 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) {
 int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) {
   if (pTask->taskLevel == TASK_LEVEL__AGG) {
     int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1);
+    qDebug("s-task:%s remain unfinished child tasks:%d", pTask->id.idStr, left);
     ASSERT(left >= 0);
     if (left == 0) {
       streamAggChildrenRecoverFinish(pTask);

From a3bf0468c373777ac944ac1b32bf148f8989153e Mon Sep 17 00:00:00 2001
From: xleili <xlli@taosdata.com>
Date: Fri, 14 Apr 2023 18:01:56 +0800
Subject: [PATCH 40/65] release: upgrade default version

---
 cmake/cmake.version        | 2 +-
 packaging/tools/makepkg.sh | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cmake/cmake.version b/cmake/cmake.version
index 5150ee3b75..232e86d891 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.3.2")
+  SET(TD_VER_NUMBER "3.0.4.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index 0dce526db6..e4df233d67 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -150,7 +150,7 @@ fi
 mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
 mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb
 mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm
-mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||:
+# mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||:
 
 if [ $adapterName != "taosadapter" ]; then
   mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml
@@ -322,6 +322,7 @@ if [[ $dbName == "taos" ]]; then
       mkdir -p ${install_dir}/share/
       cp -Rfap ${web_dir}/admin ${install_dir}/share/
       cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
+      cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||:
     else
       echo "directory not found for enterprise release: ${web_dir}/admin"
     fi

From 3ee3b16ebb86bd697ac72e4ffbb4a81696de8a10 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 14 Apr 2023 19:37:58 +0800
Subject: [PATCH 41/65] fix(query): fix the invalid read.

---
 source/libs/executor/inc/executorimpl.h | 2 +-
 source/libs/executor/src/executor.c     | 4 +---
 source/libs/executor/src/executorimpl.c | 9 +++++----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 8615c382d3..2cb6626b03 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -826,7 +826,7 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);
 void doDestroyTask(SExecTaskInfo* pTaskInfo);
 void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
 
-char* buildTaskId(uint64_t taskId, uint64_t queryId);
+void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst);
 
 SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo);
 
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 2bc91f0cb3..6e3a7d8725 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -173,9 +173,7 @@ void doSetTaskId(SOperatorInfo* pOperator) {
 void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) {
   SExecTaskInfo* pTaskInfo = tinfo;
   pTaskInfo->id.queryId = queryId;
-
-  taosMemoryFreeClear(pTaskInfo->id.str);
-  pTaskInfo->id.str = buildTaskId(taskId, queryId);
+  buildTaskId(taskId, queryId, pTaskInfo->id.str);
 
   // set the idstr for tsdbReader
   doSetTaskId(pTaskInfo->pRoot);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 6c1c3c21c3..7594079cfb 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1151,8 +1151,8 @@ void cleanupExprSupp(SExprSupp* pSupp) {
 
 void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); }
 
-char* buildTaskId(uint64_t taskId, uint64_t queryId) {
-  char* p = taosMemoryMalloc(64);
+void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst) {
+  char* p = dst;
 
   int32_t offset = 6;
   memcpy(p, "TID:0x", offset);
@@ -1163,7 +1163,6 @@ char* buildTaskId(uint64_t taskId, uint64_t queryId) {
   offset += tintToHex(queryId, &p[offset]);
 
   p[offset] = 0;
-  return p;
 }
 
 SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model,
@@ -1185,7 +1184,9 @@ SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t v
   taosInitRWLatch(&pTaskInfo->lock);
   pTaskInfo->id.vgId = vgId;
   pTaskInfo->id.queryId = queryId;
-  pTaskInfo->id.str = buildTaskId(taskId, queryId);
+
+  pTaskInfo->id.str = taosMemoryMalloc(64);
+  buildTaskId(taskId, queryId, pTaskInfo->id.str);
   return pTaskInfo;
 }
 

From 58d9f615fe1b5e245ef30bdc04535dc46278e4c2 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Fri, 14 Apr 2023 19:54:04 +0800
Subject: [PATCH 42/65] fix: udf plan error

---
 source/libs/nodes/src/nodesEqualFuncs.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c
index 4e23999ec2..156744ef1d 100644
--- a/source/libs/nodes/src/nodesEqualFuncs.c
+++ b/source/libs/nodes/src/nodesEqualFuncs.c
@@ -136,6 +136,7 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo
 
 static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) {
   COMPARE_SCALAR_FIELD(funcId);
+  COMPARE_STRING_FIELD(functionName);
   COMPARE_NODE_LIST_FIELD(pParameterList);
   return true;
 }

From 430457e5f834f77af1422b651fbd7e9c3b52416d Mon Sep 17 00:00:00 2001
From: shenglian zhou <slzhou@taosdata.com>
Date: Fri, 14 Apr 2023 20:02:46 +0800
Subject: [PATCH 43/65] fix: copy version from main branch of tmqDelete-1ctb.py

---
 tests/system-test/7-tmq/tmqDelete-1ctb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py
index 4b8c8c8629..aa9c8d25d0 100644
--- a/tests/system-test/7-tmq/tmqDelete-1ctb.py
+++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py
@@ -238,7 +238,7 @@ class TDTestCase:
 
         if self.snapshot == 0:
             consumerId     = 2
-            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2
         elif self.snapshot == 1:
             consumerId     = 3
             expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4))
@@ -324,7 +324,7 @@ class TDTestCase:
 
         if self.snapshot == 0:
             consumerId     = 4
-            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
+            expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2
         elif self.snapshot == 1:
             consumerId     = 5
             expectrowcnt   = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4))

From 01bacc10ef7800b4e9ffc0b00d0208b59e0073a2 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 14 Apr 2023 21:43:40 +0800
Subject: [PATCH 44/65] fix: an important fix

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index dd11134bd0..c6f0310599 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -1259,6 +1259,7 @@ void tBlockDataReset(SBlockData *pBlockData) {
   pBlockData->suid = 0;
   pBlockData->uid = 0;
   pBlockData->nRow = 0;
+  pBlockData->nColData = 0;
 }
 
 void tBlockDataClear(SBlockData *pBlockData) {

From bd1c8e1902f7175bc5aa2021f96386b717794e2a Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 14 Apr 2023 21:59:08 +0800
Subject: [PATCH 45/65] fix more code

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index c6f0310599..7286dcc149 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "osMemory.h"
 #include "tdataformat.h"
 #include "tsdb.h"
 
@@ -1259,7 +1260,11 @@ void tBlockDataReset(SBlockData *pBlockData) {
   pBlockData->suid = 0;
   pBlockData->uid = 0;
   pBlockData->nRow = 0;
+  for (int32_t i = 0; i < pBlockData->nColData; i++) {
+    tColDataDestroy(&pBlockData->aColData[i]);
+  }
   pBlockData->nColData = 0;
+  taosMemoryFreeClear(pBlockData->aColData);
 }
 
 void tBlockDataClear(SBlockData *pBlockData) {

From e05d61456a580c7e3c1efbdfe8542121d25feb79 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 14 Apr 2023 22:45:22 +0800
Subject: [PATCH 46/65] fix(stream): disable the status check.

---
 source/libs/stream/src/streamMeta.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index fecc01f295..d3972a19d1 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -84,10 +84,10 @@ void streamMetaClose(SStreamMeta* pMeta) {
   tdbClose(pMeta->db);
 
   void* pIter = NULL;
-  while(pMeta->walScan) {
-    qDebug("wait stream daemon quit");
-    taosMsleep(100);
-  }
+//  while(pMeta->walScan) {
+//    qDebug("wait stream daemon quit");
+//    taosMsleep(100);
+//  }
 
   while (1) {
     pIter = taosHashIterate(pMeta->pTasks, pIter);

From 3aa6897738c7ad0fecfbb7429dbfeeb976f3af20 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 14 Apr 2023 22:56:11 +0800
Subject: [PATCH 47/65] refactor: increase the buffer size

---
 source/libs/stream/src/stream.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c
index 5ec5be169e..7171b52912 100644
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@@ -16,7 +16,7 @@
 #include "streamInc.h"
 #include "ttimer.h"
 
-#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 2
+#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 2000
 
 int32_t streamInit() {
   int8_t old;

From edf9fdbecc99d2f8aa5b1054769bdc15ad3e5709 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 14 Apr 2023 23:19:36 +0800
Subject: [PATCH 48/65] make it compile

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 7286dcc149..8e778da877 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "osMemory.h"
 #include "tdataformat.h"
 #include "tsdb.h"
 

From a3b02a80c056820be35acd861e8cdaa54b4218b3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 15 Apr 2023 00:10:08 +0800
Subject: [PATCH 49/65] fix(stream): update the version when open stream tasks.

---
 source/libs/stream/src/streamMeta.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index d3972a19d1..51cc315780 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -296,7 +296,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
     tDecodeStreamTask(&decoder, pTask);
     tDecoderClear(&decoder);
 
-    if (pMeta->expandFunc(pMeta->ahandle, pTask, -1) < 0) {
+    // todo set correct initial version.
+    if (pMeta->expandFunc(pMeta->ahandle, pTask, 0) < 0) {
       tdbFree(pKey);
       tdbFree(pVal);
       tdbTbcClose(pCur);

From 1a0c9f31bc105b7e8e1f8ffeb10b539be7b7a1e2 Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Mon, 17 Apr 2023 10:34:34 +0800
Subject: [PATCH 50/65] enhance: refactor cleanup udf function

---
 source/libs/function/src/tudf.c | 72 +++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 30 deletions(-)

diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index f13235f24b..8c8b99a6f8 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -968,6 +968,8 @@ bool    udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pRes
 int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
 int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
 
+void cleanupNotExpiredUdfs();
+void cleanupExpiredUdfs();
 int compareUdfcFuncSub(const void *elem1, const void *elem2) {
   SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1;
   SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2;
@@ -1040,18 +1042,32 @@ void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) {
   uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
 }
 
-int32_t cleanUpUdfs() {
-  int8_t initialized = atomic_load_8(&gUdfcProxy.initialized);
-  if (!initialized) {
-    return TSDB_CODE_SUCCESS;
+void cleanupExpiredUdfs() {
+  int32_t i = 0;
+  SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
+  while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) {
+    SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i);
+    if (stub->refCount == 0) {
+      fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount);
+      doTeardownUdf(stub->handle);
+    } else {
+      fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName,
+             stub->refCount, stub->createTime, stub->handle);
+      UdfcFuncHandle handle = stub->handle;
+      if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
+        taosArrayPush(expiredUdfStubs, stub);
+      } else {
+        fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache",
+               stub->udfName, stub->refCount, stub->createTime);
+      }
+    }
+    ++i;
   }
+  taosArrayDestroy(gUdfcProxy.expiredUdfStubs);
+  gUdfcProxy.expiredUdfStubs = expiredUdfStubs;
+}
 
-  uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
-  if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) &&
-      (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) {
-    uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
-    return TSDB_CODE_SUCCESS;
-  }
+void cleanupNotExpiredUdfs() {
   SArray *udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
   int32_t i = 0;
   while (i < taosArrayGetSize(gUdfcProxy.udfStubs)) {
@@ -1074,28 +1090,24 @@ int32_t cleanUpUdfs() {
   }
   taosArrayDestroy(gUdfcProxy.udfStubs);
   gUdfcProxy.udfStubs = udfStubs;
+}
 
-  SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
-  while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) {
-    SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i);
-    if (stub->refCount == 0) {
-      fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount);
-      doTeardownUdf(stub->handle);
-    } else {
-      fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName,
-             stub->refCount, stub->createTime, stub->handle);
-      UdfcFuncHandle handle = stub->handle;
-      if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
-        taosArrayPush(expiredUdfStubs, stub);
-      } else {
-        fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache",
-               stub->udfName, stub->refCount, stub->createTime);
-      }
-    }
-    ++i;
+int32_t cleanUpUdfs() {
+  int8_t initialized = atomic_load_8(&gUdfcProxy.initialized);
+  if (!initialized) {
+    return TSDB_CODE_SUCCESS;
   }
-  taosArrayDestroy(gUdfcProxy.expiredUdfStubs);
-  gUdfcProxy.expiredUdfStubs = expiredUdfStubs;
+
+  uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
+  if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) &&
+      (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) {
+    uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  cleanupNotExpiredUdfs();
+  cleanupExpiredUdfs();
+
   uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
   return 0;
 }

From 2c52b00070c29113ba9eefb3654aec4f0b04814c Mon Sep 17 00:00:00 2001
From: sunpeng <sunpeng.dev@gmail.com>
Date: Mon, 17 Apr 2023 11:49:10 +0800
Subject: [PATCH 51/65] docs update offcial doc monitor section (#20949)

---
 docs/zh/17-operation/10-monitor.md | 299 +++++++++++++++++++++++++++++
 1 file changed, 299 insertions(+)

diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md
index e936f35dca..b31ada1b79 100644
--- a/docs/zh/17-operation/10-monitor.md
+++ b/docs/zh/17-operation/10-monitor.md
@@ -38,3 +38,302 @@ chmod +x TDinsight.sh
 运行程序并重启 Grafana 服务，打开面板：`http://localhost:3000/d/tdinsight`。
 
 更多使用场景和限制请参考[TDinsight](/reference/tdinsight/) 文档。
+
+## log 库
+
+TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](..../reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。
+
+### cluster info 表
+
+`cluster_info` 表记录集群信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|first\_ep|VARCHAR||集群 first ep|
+|first\_ep\_dnode\_id|INT||集群 first ep 的 dnode id|
+|version|VARCHAR||tdengine version。例如：3.0.4.0|
+|master\_uptime|FLOAT||当前 master 节点的uptime。单位：天|
+|monitor_interval|INT||monitor interval。单位：秒|
+|dbs\_total|INT||database 总数|
+|tbs\_total|BIGINT||当前集群 table 总数|
+|stbs\_total|INT||当前集群 stable 总数|
+|dnodes\_total|INT||当前集群 dnode 总数|
+|dnodes\_alive|INT||当前集群 dnode 存活总数|
+|mnodes\_total|INT||当前集群 mnode 总数|
+|mnodes\_alive|INT||当前集群 mnode 存活总数|
+|vgroups\_total|INT||当前集群 vgroup 总数|
+|vgroups\_alive|INT||当前集群 vgroup 存活总数|
+|vnodes\_total|INT||当前集群 vnode 总数|
+|vnodes\_alive|INT||当前集群 vnode 存活总数|
+|connections\_total|INT||当前集群连接总数|
+|topics\_total|INT||当前集群 topic 总数|
+|streams\_total|INT||当前集群 stream 总数|
+|protocol|INT||协议版本，目前为 1|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### d info 表
+
+`d_info` 表记录 dnode 状态信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|status|VARCHAR||dnode 状态|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### m info 表
+
+`m_info` 表记录 mnode 角色信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|role|VARCHAR||mnode 角色， leader 或 follower|
+|mnode\_id|INT|TAG|master node id|
+|mnode\_ep|NCHAR|TAG|master node endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### dnodes info 表
+
+`dnodes_info` 记录 dnode 信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|uptime|FLOAT||dnode uptime|
+|cpu\_engine|FLOAT||taosd cpu 使用率，从 `/proc/<taosd_pid>/stat` 读取|
+|cpu\_system|FLOAT||服务器 cpu 使用率，从 `/proc/stat` 读取|
+|cpu\_cores|FLOAT||服务器 cpu 核数|
+|mem\_engine|INT||taosd 内存使用率，从 `/proc/<taosd_pid>/status` 读取|
+|mem\_system|INT||服务器内存使用率|
+|mem\_total|INT||服务器内存总量，单位 KB|
+|disk\_engine|INT|||
+|disk\_used|BIGINT||data dir 挂载的磁盘使用量，单位 bytes|
+|disk\_total|BIGINT||data dir 挂载的磁盘总容量，单位 bytes|
+|net\_in|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 bytes per second|
+|net\_out|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 bytes per second|
+|io\_read|FLOAT||io 吞吐率，从 `/proc/<taosd_pid>/io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 bytes per second|
+|io\_write|FLOAT||io 吞吐率，从 `/proc/<taosd_pid>/io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 bytes per second|
+|io\_read\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc/<taosd_pid>/io` 中读取的 read_bytes。单位 bytes per second|
+|io\_write\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc/<taosd_pid>/io` 中读取的 write_bytes。单位 bytes per second|
+|req\_select|INT||两个间隔内发生的查询请求数目|
+|req\_select\_rate|FLOAT||两个间隔内的查询请求速度 = `req_select / monitorInterval`|
+|req\_insert|INT||两个间隔内发生的写入请求，包含的单条数据数目|
+|req\_insert\_success|INT||两个间隔内发生的处理成功的写入请求，包含的单条数据数目|
+|req\_insert\_rate|FLOAT||两个间隔内的单条数据写入速度 = `req_insert / monitorInterval`|
+|req\_insert\_batch|INT||两个间隔内发生的写入请求数目|
+|req\_insert\_batch\_success|INT||两个间隔内发生的成功的批量写入请求数目|
+|req\_insert\_batch\_rate|FLOAT||两个间隔内的写入请求数目的速度 = `req_insert_batch / monitorInterval`|
+|errors|INT||两个间隔内的出错的写入请求的数目|
+|vnodes\_num|INT||dnode 上 vnodes 数量|
+|masters|INT||dnode 上 master node 数量|
+|has\_mnode|INT||dnode 是否包含 mnode|
+|has\_qnode|INT||dnode 是否包含 qnode|
+|has\_snode|INT||dnode 是否包含 snode|
+|has\_bnode|INT||dnode 是否包含 bnode|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### data dir 表
+
+`data_dir` 表记录 data 目录信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|name|NCHAR||data 目录，一般为 `/var/lib/taos`|
+|level|INT||0、1、2 多级存储级别|
+|avail|BIGINT||data 目录可用空间|
+|used|BIGINT||data 目录已使用空间|
+|total|BIGINT||data 目录空间|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### log dir 表
+
+`log_dir` 表记录 log 目录信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|name|NCHAR||log 目录名，一般为 `/var/log/taos/`|
+|avail|BIGINT||log 目录可用空间|
+|used|BIGINT||log 目录已使用空间|
+|total|BIGINT||log 目录空间|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### tmp dir 表
+
+`temp_dir` 表记录 temp 目录信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|name|NCHAR||temp 目录名，一般为 `/tmp/`|
+|avail|BIGINT||temp 目录可用空间|
+|used|BIGINT||temp 目录已使用空间|
+|total|BIGINT||temp 目录空间|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### vgroups info 表
+
+`vgroups_info` 表记录虚拟节点组信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|vgroup\_id|INT||vgroup id|
+|database\_name|VARCHAR||vgroup 所属的 database 名字|
+|tables\_num|BIGINT||vgroup 中 table 数量|
+|status|VARCHAR||vgroup 状态|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### vnodes role 表
+
+`vnodes_role` 表记录虚拟节点角色信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|vnode\_role|VARCHAR||vnode 角色，leader 或 follower|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### logs 表
+
+`logs` 表记录登录信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|level|VARCHAR||log level|
+|content|NCHAR||log content，长度不超过1024字节|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### log summary 表
+
+`log_summary` 记录日志统计信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|error|INT||error 总数|
+|info|INT||info 总数|
+|debug|INT||debug 总数|
+|trace|INT||trace 总数|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### grants info 表
+
+`grants_info` 记录授权信息。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|expire\_time|BIGINT||认证过期时间，企业版有效，社区版为 bigint 最大值|
+|timeseries\_used|BIGINT||已用测点数|
+|timeseries\_total|BIGINT||总测点数，开源版本为 bigint 最大值|
+|dnode\_id|INT|TAG|dnode id|
+|dnode\_ep|NCHAR|TAG|dnode endpoint|
+|cluster\_id|NCHAR|TAG|cluster id|
+
+### keeper monitor 表
+
+`keeper_monitor` 记录 taoskeeper 监控数据。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|ts|TIMESTAMP||timestamp|
+|cpu|FLOAT||cpu 使用率|
+|mem|FLOAT||内存使用率|
+|identify|NCHAR|TAG||
+
+### taosadapter restful http request total 表
+
+`taosadapter_restful_http_request_total` 记录 taosadapter rest 请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|\_ts|TIMESTAMP||timestamp|
+|guage|DOUBLE||监控指标值|
+|client\_ip|NCHAR|TAG|client ip|
+|endpoint|NCHAR|TAG|taosadpater endpoint|
+|request\_method|NCHAR|TAG|request method|
+|request\_uri|NCHAR|TAG|request uri|
+|status\_code|NCHAR|TAG|status code|
+
+### taosadapter restful http request fail 表
+
+`taosadapter_restful_http_request_fail` 记录 taosadapter rest 请求失败信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|\_ts|TIMESTAMP||timestamp|
+|guage|DOUBLE||监控指标值|
+|client\_ip|NCHAR|TAG|client ip|
+|endpoint|NCHAR|TAG|taosadpater endpoint|
+|request\_method|NCHAR|TAG|request method|
+|request\_uri|NCHAR|TAG|request uri|
+|status\_code|NCHAR|TAG|status code|
+
+### taosadapter restful http request in flight 表
+
+`taosadapter_restful_http_request_in_flight` 记录 taosadapter rest 实时请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|\_ts|TIMESTAMP||timestamp|
+|guage|DOUBLE||监控指标值|
+|endpoint|NCHAR|TAG|taosadpater endpoint|
+
+### taosadapter restful http request summary milliseconds 表
+
+`taosadapter_restful_http_request_summary_milliseconds` 记录 taosadapter rest 请求汇总信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|\_ts|TIMESTAMP||timestamp|
+|count|DOUBLE|||
+|sum|DOUBLE|||
+|0.5|DOUBLE|||
+|0.9|DOUBLE|||
+|0.99|DOUBLE|||
+|0.1|DOUBLE|||
+|0.2|DOUBLE|||
+|endpoint|NCHAR|TAG|taosadpater endpoint|
+|request\_method|NCHAR|TAG|request method|
+|request\_uri|NCHAR|TAG|request uri|
+
+### taosadapter system mem percent 表
+
+`taosadapter_system_mem_percent` 表记录 taosadapter 内存使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|\_ts|TIMESTAMP||timestamp|
+|guage|DOUBLE||监控指标值|
+|endpoint|NCHAR|TAG|taosadpater endpoint|
+
+### taosadapter system cpu percent 表
+
+|field|type|is\_tag|comment|
+|:----|:---|:-----|:------|
+|\_ts|TIMESTAMP||timestamp|
+|guage|DOUBLE||监控指标值|
+|endpoint|NCHAR|TAG|taosadpater endpoint|
+

From a2d75a327cb68a4f59c76c6d2b9a2150880b6eee Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Tue, 18 Apr 2023 08:28:12 +0800
Subject: [PATCH 52/65] enhance: udf output column reserve capacity

---
 source/libs/function/src/udfd.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index 61c6f92e2f..5034af2f82 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -688,6 +688,8 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
       output.colMeta.type = udf->outputType;
       output.colMeta.precision = 0;
       output.colMeta.scale = 0;
+      udfColEnsureCapacity(&output, call->block.info.rows);
+
       SUdfDataBlock input = {0};
       convertDataBlockToUdfDataBlock(&call->block, &input);
       code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx);

From 76c628c8ee357fd753fa975d48bcf46bf002be00 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 18 Apr 2023 09:22:49 +0800
Subject: [PATCH 53/65] fix: atoi on int64 config item (#20946)

---
 source/util/src/tconfig.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index f811d2f203..288ea6052b 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -187,7 +187,7 @@ static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType st
 }
 
 static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
-  int64_t ival = (int64_t)atoi(value);
+  int64_t ival = (int64_t)atoll(value);
   if (ival < pItem->imin || ival > pItem->imax) {
     uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
            cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax);

From 9581dcae7f322716c0dcf187788d42bc520e7300 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 18 Apr 2023 14:17:22 +0800
Subject: [PATCH 54/65] enh: add param for to_unixtimestamp to return timestamp
 type

---
 source/libs/function/src/builtins.c | 27 ++++++++++++++++++++++++---
 source/libs/scalar/src/sclfunc.c    |  3 ++-
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index a293f45238..fe98a1dd53 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -1933,14 +1933,35 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
 }
 
 static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  int16_t resType = TSDB_DATA_TYPE_BIGINT;
+
+  if (1 != numOfParams && 2 != numOfParams) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  if (!IS_STR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) {
+  uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (!IS_STR_DATA_TYPE(para1Type)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  if (2 == numOfParams) {
+    uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
+    if (!IS_INTEGER_TYPE(para2Type)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
+    if (pValue->datum.i == 1) {
+      resType = TSDB_DATA_TYPE_TIMESTAMP;
+    } else if (pValue->datum.i == 0) {
+      resType = TSDB_DATA_TYPE_BIGINT;
+    } else {
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                             "TO_UNIXTIMESTAMP function second parameter should be 0/1");
+    }
+  }
+
   // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
   int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
@@ -1948,7 +1969,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int
     return code;
   }
 
-  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType};
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 24b25cec80..835264eabe 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1124,7 +1124,8 @@ _end:
 int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   int32_t type = GET_PARAM_TYPE(pInput);
   int64_t timePrec;
-  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
+  int32_t idx = (inputNum == 2) ? 1 : 2;
+  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[idx]), pInput[idx].columnData->pData);
 
   for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
     if (colDataIsNull_s(pInput[0].columnData, i)) {

From 56ab728a4213b9767a4548e9bd24d27cdbf89a8d Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 18 Apr 2023 14:18:06 +0800
Subject: [PATCH 55/65] add test cases

---
 tests/system-test/2-query/To_unixtimestamp.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py
index 8ee2007450..00a8ed84de 100644
--- a/tests/system-test/2-query/To_unixtimestamp.py
+++ b/tests/system-test/2-query/To_unixtimestamp.py
@@ -26,7 +26,7 @@ class TDTestCase:
             'c1':'int',
             'c2':'float',
             'c3':'binary(20)'
-            
+
         }
         # structure of tag
         self.tag_dict = {
@@ -60,7 +60,7 @@ class TDTestCase:
             if tb_type == 'ntb' or tb_type == 'ctb':
                 tdSql.checkRows(len(values_list))
             elif tb_type == 'stb':
-                tdSql.checkRows(len(self.values_list)*tb_num)    
+                tdSql.checkRows(len(self.values_list)*tb_num)
         for time in ['2020-01-32T08:00:00','2020-13-32T08:00:00','acd']:
             tdSql.query(f"select to_unixtimestamp('{time}') from {tbname}")
             if tb_type == 'ntb' or tb_type == 'ctb':
@@ -74,7 +74,7 @@ class TDTestCase:
             if tb_type == 'ntb' or tb_type == 'ctb':
                 tdSql.checkRows(len(values_list))
             elif tb_type == 'stb':
-                tdSql.checkRows(len(values_list)*tb_num)    
+                tdSql.checkRows(len(values_list)*tb_num)
         for time in self.error_param:
             tdSql.error(f"select to_unixtimestamp({time}) from {tbname}")
     def timestamp_change_check_ntb(self):
@@ -95,9 +95,20 @@ class TDTestCase:
             self.data_check(f'{self.stbname}_{i}',self.values_list,'ctb')
         self.data_check(self.stbname,self.values_list,'stb',self.tbnum)
         tdSql.execute(f'drop database {self.dbname}')
+    def timestamp_change_return_type(self):
+        tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 0);")
+        tdSql.checkEqual(tdSql.queryResult[0][0], 0)
+        tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1);")
+        tdSql.checkEqual(tdSql.queryResult[0][0], datetime.datetime(1970, 1, 1, 8, 0, 0))
+        tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 2);")
+        tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1.5);")
+        tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 'abc');")
+        tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', true);")
+        tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1, 3);")
     def run(self):  # sourcery skip: extract-duplicate-method
         self.timestamp_change_check_ntb()
         self.timestamp_change_check_stb()
+        self.timestamp_change_return_type()
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

From b2cca108a349ae4a5fa10eee560734f04a35a56d Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 18 Apr 2023 14:29:04 +0800
Subject: [PATCH 56/65] add documentations

---
 docs/en/12-taos-sql/10-function.md | 10 ++++++++--
 docs/zh/12-taos-sql/10-function.md | 10 ++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md
index 8dfa9c2851..5c1a833e05 100644
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@@ -459,12 +459,17 @@ TO_JSON(str_literal)
 #### TO_UNIXTIMESTAMP
 
 ```sql
-TO_UNIXTIMESTAMP(expr)
+TO_UNIXTIMESTAMP(expr [, return_timestamp])
+
+return_timestamp: {
+    0
+  | 1
+}
 ```
 
 **Description**: UNIX timestamp converted from a string of date/time format
 
-**Return value type**: BIGINT
+**Return value type**: BIGINT, TIMESTAMP
 
 **Applicable column types**: VARCHAR and NCHAR
 
@@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr)
 
 - The input string must be compatible with ISO8601/RFC3339 standard, NULL will be returned if the string can't be converted
 - The precision of the returned timestamp is same as the precision set for the current data base in use
+- return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT.
 
 
 ### Time and Date Functions
diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md
index 94f8052051..50e82e6b90 100644
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@@ -459,12 +459,17 @@ TO_JSON(str_literal)
 #### TO_UNIXTIMESTAMP
 
 ```sql
-TO_UNIXTIMESTAMP(expr)
+TO_UNIXTIMESTAMP(expr [, return_timestamp])
+
+return_timestamp: {
+    0
+  | 1
+}
 ```
 
 **功能说明**：将日期时间格式的字符串转换成为 UNIX 时间戳。
 
-**返回结果数据类型**：BIGINT。
+**返回结果数据类型**：BIGINT, TIMESTAMP。
 
 **应用字段**：VARCHAR, NCHAR。
 
@@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr)
 
 - 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。
 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。
+- return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。
 
 
 ### 时间和日期函数

From 1d2764ebdb50b66673219672a2062dadae846730 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Tue, 18 Apr 2023 15:36:50 +0800
Subject: [PATCH 57/65] feat: the fill value clause supports constant
 expressions

---
 source/libs/parser/inc/sql.y           |    4 +-
 source/libs/parser/src/parTranslater.c |   22 +-
 source/libs/parser/src/sql.c           | 1292 ++++++++++++------------
 3 files changed, 678 insertions(+), 640 deletions(-)

diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y
index f99976e0df..d7a6baaffe 100644
--- a/source/libs/parser/inc/sql.y
+++ b/source/libs/parser/inc/sql.y
@@ -1047,8 +1047,8 @@ sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP.
 
 fill_opt(A) ::= .                                                                 { A = NULL; }
 fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP.                                    { A = createFillNode(pCxt, B, NULL); }
-fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP.                  { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); }  
-fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA literal_list(B) NK_RP.                { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); }  
+fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP.               { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); }
+fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP.             { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); }
 
 %type fill_mode                                                                   { EFillMode }
 %destructor fill_mode                                                             { }
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index e29be53bb6..cbe7f5668c 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -1310,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
 }
 
 static EDealRes haveVectorFunction(SNode* pNode, void* pContext) {
-  if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || isInterpPseudoColumnFunc(pNode)) {
+  if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) ||
+      isInterpPseudoColumnFunc(pNode)) {
     *((bool*)pContext) = true;
     return DEAL_RES_END;
   }
@@ -2911,6 +2912,11 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList
   if (TSDB_CODE_SUCCESS == code) {
     code = scalarCalculateConstants(pCaseFunc, &pCell->pNode);
   }
+  if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) {
+    code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value is just a constant");
+  } else if (TSDB_CODE_SUCCESS != code) {
+    code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch");
+  }
   return code;
 }
 
@@ -2927,9 +2933,9 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL
       if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) {
         return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch");
       }
-      if (TSDB_CODE_SUCCESS !=
-          convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo)) {
-        return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch");
+      int32_t code = convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo);
+      if (TSDB_CODE_SUCCESS != code) {
+        return code;
       }
       ++fillNo;
     }
@@ -5715,6 +5721,14 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt
 
 static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) {
   // The statement is executed directly on the client without constructing a message.
+  if ('\0' != pStmt->value[0]) {
+    return TSDB_CODE_SUCCESS;
+  }
+  char* p = strchr(pStmt->config, ' ');
+  if (NULL != p) {
+    *p = 0;
+    strcpy(pStmt->value, p + 1);
+  }
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c
index 9ad5bcf644..a38e2368dd 100644
--- a/source/libs/parser/src/sql.c
+++ b/source/libs/parser/src/sql.c
@@ -217,331 +217,343 @@ typedef union {
 **  yy_default[]       Default action for each state.
 **
 *********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (2905)
+#define YY_ACTTAB_COUNT (3030)
 static const YYACTIONTYPE yy_action[] = {
- /*     0 */  2116, 1881, 2010,  499,  432, 1877,  500, 1758,  431, 2102,
- /*    10 */   670, 2051,   46,   44, 1646, 1722, 2102, 2008,  640, 2098,
- /*    20 */   393,  504, 1495, 1520,   39,   38, 2098,  501,   45,   43,
- /*    30 */    42,   41,   40, 1576, 1791, 1493, 2134, 2010, 1520,  132,
- /*    40 */   131,  130,  129,  128,  127,  126,  125,  124, 2084,  384,
- /*    50 */   669,  591, 2007,  640, 2275, 2094, 2100,  374,  244, 1571,
- /*    60 */    28, 1944, 2094, 2100,  375,   19,  663,  652,  371, 2281,
- /*    70 */   184,  638, 1501,  663, 2276,  617, 1942,  628,  140, 1869,
- /*    80 */   507, 2115,  107,  500, 1758, 2151,   36,  296,  169, 2117,
- /*    90 */   673, 2119, 2120,  668,  168,  663, 1734,  758,  141,    9,
+ /*     0 */  2116,  209, 2010, 1881,  432,  502, 1883, 1765,  431, 2102,
+ /*    10 */   670, 2010,   48,   46, 1646, 1722, 2098, 2008,  640, 2098,
+ /*    20 */   393, 1522, 1495,  384,   41,   40, 2007,  640,   47,   45,
+ /*    30 */    44,   43,   42, 1576, 1791, 1493, 2134,  180, 1520,  132,
+ /*    40 */   131,  130,  129,  128,  127,  126,  125,  124, 2084, 1931,
+ /*    50 */   669,  591, 2094, 2100, 2275, 2094, 2100,  388,  398, 1571,
+ /*    60 */    30, 1937, 1939,  663,  499,   19,  663,  500, 1758, 2281,
+ /*    70 */   184,  168, 1501, 1734, 2276,  617, 1520,  628,  140, 1869,
+ /*    80 */   507, 2115,  107,  500, 1758, 2151,   38,  296,  169, 2117,
+ /*    90 */   673, 2119, 2120,  668,  655,  663, 2176,  758,  141,    9,
  /*   100 */    15,  735,  734,  733,  732,  403, 1884,  731,  730,  144,
  /*   110 */   725,  724,  723,  722,  721,  720,  719,  157,  715,  714,
  /*   120 */   713,  402,  401,  710,  709,  708,  707,  706,  592, 2241,
- /*   130 */   398, 2219, 1320, 1937, 1939,  123, 1578, 1579,  122,  121,
+ /*   130 */    62, 2219, 1320, 1938, 1939,  123, 1578, 1579,  122,  121,
  /*   140 */   120,  119,  118,  117,  116,  115,  114, 1311,  695,  694,
  /*   150 */   693, 1315,  692, 1317, 1318,  691,  688, 2216, 1326,  685,
- /*   160 */  1328, 1329,  682,  679,  177,  652, 1551, 1561, 2280, 1409,
- /*   170 */  1410, 2275, 1577, 1580, 1944,  653, 1892,  630,  182, 2212,
- /*   180 */  2213,  356,  138, 2217,  358, 1993, 1496, 2279, 1494, 1942,
- /*   190 */  1720, 2276, 2278,  133,  287,  288,  516,   39,   38,  286,
- /*   200 */   537,   45,   43,   42,   41,   40,  278,   62,  703,  155,
+ /*   160 */  1328, 1329,  682,  679,  504,  652, 1551, 1561, 2280,  277,
+ /*   170 */   501, 2275, 1577, 1580, 1944,  653, 1892,  630,  182, 2212,
+ /*   180 */  2213,  356,  138, 2217, 1409, 1410, 1496, 2279, 1494, 1942,
+ /*   190 */  1720, 2276, 2278,  133,  287,  288, 1989,   41,   40,  286,
+ /*   200 */   537,   47,   45,   44,   43,   42,  278,   52,  703,  155,
  /*   210 */   154,  700,  699,  698,  152, 1499, 1500, 1794, 1550, 1553,
  /*   220 */  1554, 1555, 1556, 1557, 1558, 1559, 1560,  665,  661, 1569,
- /*   230 */  1570, 1572, 1573, 1574, 1575,    2,   46,   44,  425, 1169,
- /*   240 */  1522,  341,   62, 1518,  393,   49, 1495,   62,  611,   93,
+ /*   230 */  1570, 1572, 1573, 1574, 1575,    2,   48,   46,  425, 1169,
+ /*   240 */   192,  341,   62, 1518,  393,   51, 1495,   62,   87,   93,
  /*   250 */   469, 2116,  616,  483,  350, 2275,  482, 1576,  177, 1493,
- /*   260 */   406,  670,  427,  423,  405,   45,   43,   42,   41,   40,
- /*   270 */   615,  184,  452,   50,  484, 2276,  617,  454, 1171, 1994,
- /*   280 */  1174, 1175,  180, 1571,  555,  554,  553, 2134, 1723,   19,
- /*   290 */   106,  545,  137,  549, 1931, 1520, 1501,  548, 1605, 2084,
+ /*   260 */   406,  670,  427,  423,  405,   47,   45,   44,   43,   42,
+ /*   270 */   615,  184,  452, 1888,  484, 2276,  617,  454, 1171, 1994,
+ /*   280 */  1174, 1175,  187, 1571,  555,  554,  553, 2134, 1723,   19,
+ /*   290 */   106,  545,  137,  549,  551,  550, 1501,  548, 1605, 2084,
  /*   300 */   103,  669,  547,  552,  366,  365, 1521,  591,  546,  123,
  /*   310 */  2275, 1639,  122,  121,  120,  119,  118,  117,  116,  115,
  /*   320 */   114,  758,  359,  101,   15, 2281,  184,  430, 2280,  429,
  /*   330 */  2276,  617, 2115, 1191,  442, 1190, 2151,  653, 1892,  110,
  /*   340 */  2117,  673, 2119, 2120,  668, 1522,  663, 1885,  226,  143,
  /*   350 */   438,  150, 2175, 2204, 1606,  133,  428,  387, 2200,  187,
- /*   360 */  1578, 1579,  542,  480, 1519, 1192,  474,  473,  472,  471,
+ /*   360 */  1578, 1579,  542,  480,  652, 1192,  474,  473,  472,  471,
  /*   370 */   468,  467,  466,  465,  464,  460,  459,  458,  457,  340,
- /*   380 */   449,  448,  447,  209,  444,  443,  357,  502,  277, 1765,
- /*   390 */  1551, 1561, 2280,  338,  187, 2275, 1577, 1580, 1650,  187,
- /*   400 */   555,  554,  553,  705, 1520, 1938, 1939,  545,  137,  549,
- /*   410 */  1496, 2279, 1494,  548,  606, 2276, 2277, 1868,  547,  552,
- /*   420 */   366,  365, 1354, 1355,  546,  187, 1708, 1264,   35,  391,
+ /*   380 */   449,  448,  447,  177,  444,  443,  357,  653, 1892,  638,
+ /*   390 */  1551, 1561, 2280,  338,  187, 2275, 1577, 1580,  165,  187,
+ /*   400 */   555,  554,  553,  358, 1993,  189, 1895,  545,  137,  549,
+ /*   410 */  1496, 2279, 1494,  548,  606, 2276, 2277, 2070,  547,  552,
+ /*   420 */   366,  365, 1354, 1355,  546, 2279, 1708, 1264,   37,  391,
  /*   430 */  1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499,
- /*   440 */  1500, 1552, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559,
+ /*   440 */  1500, 1974, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559,
  /*   450 */  1560,  665,  661, 1569, 1570, 1572, 1573, 1574, 1575,    2,
- /*   460 */    12,   46,   44,  227, 1266, 1495,  167, 1521,  211,  393,
- /*   470 */  2116, 1495,  502, 1833, 1765, 1674,  578,  652, 1493,  172,
- /*   480 */   631,  705, 1576,  476, 1493,  533,  529,  525,  521,  224,
- /*   490 */  2219,   39,   38,  277, 1745,   45,   43,   42,   41,   40,
- /*   500 */   612,  607,  600, 1523, 1523,   66, 2134, 1883, 1571,  551,
- /*   510 */   550, 1501, 1520,  610,   19, 1501, 2215, 2098, 2084,  639,
+ /*   460 */    12,   48,   46,  227, 1266, 1495,  413, 1521,  610,  393,
+ /*   470 */  2116, 1495,  652,  196,  167, 1674,  516, 1520, 1493,  172,
+ /*   480 */   631, 1833, 1576,  476, 1493,  533,  529,  525,  521,  224,
+ /*   490 */   253,   41,   40,  277, 2134,   47,   45,   44,   43,   42,
+ /*   500 */   612,  607,  600, 1523,  611,   66, 2134, 2103, 1571,   12,
+ /*   510 */  1191,   10, 1190,   84,   19, 1501,   83, 2098, 2084,  639,
  /*   520 */   669, 1501,  603,  602, 1672, 1673, 1675, 1676, 1677,   88,
- /*   530 */    39,   38,  222,   12,   45,   43,   42,   41,   40, 2134,
- /*   540 */   758, 1944, 2084,  200,  199,  543,  758, 1744,  381,   15,
- /*   550 */  1552, 2115, 1607, 2094, 2100, 2151, 1942,   49,  110, 2117,
- /*   560 */   673, 2119, 2120,  668,  663,  663,  475, 1262,  166,  514,
- /*   570 */   181, 2003, 2204,  316,   39,   38,  387, 2200,   45,   43,
- /*   580 */    42,   41,   40, 1426, 1427, 1578, 1579,  314,   73,  186,
- /*   590 */  1743,   72,   62,  609,  560, 2084, 1191, 2230, 1190,  221,
- /*   600 */   215,   62,   87,  639,  220,   12,  512,   10, 2219,  570,
- /*   610 */   207,  495,  493,  490,  696, 1551, 1561,  396,  360, 1425,
- /*   620 */  1428, 1577, 1580,  240,  213,  162,   33, 1887, 1192, 1496,
- /*   630 */   717, 1494,  372, 1894, 2214, 1496, 1612, 1494, 2084,  563,
- /*   640 */  1942, 2067,  653, 1892,  557,  653, 1892, 1944, 1742,  239,
- /*   650 */    62,  255,  193,  637,  386, 2003, 2102,  541, 1499, 1500,
- /*   660 */   189,  540, 1942,   55, 1499, 1500, 2098, 1550, 1553, 1554,
+ /*   530 */    41,   40,  222,   91,   47,   45,   44,   43,   42, 1868,
+ /*   540 */   758,  211, 1192,  200,  199,  502,  758, 1765,  609,   15,
+ /*   550 */  1650, 2115,   51, 2094, 2100, 2151, 1520,   62,  110, 2117,
+ /*   560 */   673, 2119, 2120,  668,  663,  663,  475, 2051,  166,  514,
+ /*   570 */   181, 2003, 2204,  316,   41,   40,  387, 2200,   47,   45,
+ /*   580 */    44,   43,   42, 1426, 1427, 1578, 1579,  314,   73,  186,
+ /*   590 */   657,   72, 2176,  718,  560, 1854,  456, 2230, 1745,  221,
+ /*   600 */   215,   62,  705,  705,  220,  455,  512,   14,   13,  570,
+ /*   610 */   207,  495,  493,  490,  244, 1551, 1561,  396,  385, 1425,
+ /*   620 */  1428, 1577, 1580,  240,  213,  162,  165,  653, 1892, 1496,
+ /*   630 */  1177, 1494,  372, 1894, 1894, 1496, 1519, 1494,  616,  563,
+ /*   640 */  1942, 2275,  653, 1892,  557,   57, 2084, 1519, 1944,  239,
+ /*   650 */    62,  255,   44,   43,   42,  371,  615,  184, 1499, 1500,
+ /*   660 */   436, 2276,  617, 1942, 1499, 1500, 1523, 1550, 1553, 1554,
  /*   670 */  1555, 1556, 1557, 1558, 1559, 1560,  665,  661, 1569, 1570,
- /*   680 */  1572, 1573, 1574, 1575,    2,   46,   44, 1581,  109,   70,
- /*   690 */  1944, 2116,   69,  393, 1879, 1495, 2084,  397,  653, 1892,
- /*   700 */  1715,  631, 2094, 2100,  388, 1942, 1576, 1741, 1493,  187,
- /*   710 */   718,   32, 1854,  663,  628,  140,  436,   39,   38,  653,
- /*   720 */  1892,   45,   43,   42,   41,   40, 1834, 2134,   81,   80,
- /*   730 */   435,   87, 1571,  191,  164,  729,  727,  437,  639, 2084,
- /*   740 */  1875,  669,   39,   38,  187, 1501,   45,   43,   42,   41,
- /*   750 */    40,  653, 1892,  187,  339, 2084, 1888,  421, 1974, 1989,
- /*   760 */   419,  415,  411,  408,  428, 1177, 1740,  653, 1892,  446,
- /*   770 */   758, 1519, 2115,   47,  653, 1892, 2151, 2116, 1739,  110,
- /*   780 */  2117,  673, 2119, 2120,  668,  461,  663,  670,  648, 1767,
- /*   790 */  2003,  181,  462, 2204,   34, 1643, 1738,  387, 2200, 1714,
- /*   800 */    39,   38,  187,  192,   45,   43,   42,   41,   40, 1578,
- /*   810 */  1579,  653, 1892, 2134, 2084,  275, 2212,  627, 2231,  134,
- /*   820 */   626,  569, 2275,  628,  140, 2084, 2084,  669, 1896,  515,
- /*   830 */  1523,  653, 1892,  196,  567, 1685,  565,  615,  184, 1551,
- /*   840 */  1561, 1989, 2276,  617, 2084, 1577, 1580,   39,   38, 1889,
- /*   850 */   420,   45,   43,   42,   41,   40, 1867,  142, 2115, 1496,
- /*   860 */  2175, 1494, 2151,  653, 1892,  110, 2117,  673, 2119, 2120,
- /*   870 */   668, 1944,  663,   84,  242,  249,   83, 2295,  241, 2204,
- /*   880 */   363,  245,  165,  387, 2200,  194, 1943,  634, 1499, 1500,
- /*   890 */  1895, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560,
- /*   900 */   665,  661, 1569, 1570, 1572, 1573, 1574, 1575,    2,   46,
- /*   910 */    44, 2116,  655,  456, 2176,  653, 1892,  393, 1737, 1495,
- /*   920 */  1736,  670,  455, 2238,  183, 2212, 2213,  253,  138, 2217,
- /*   930 */  1576, 2116, 1493,  587,  591, 1174, 1175, 2275,   90,  345,
- /*   940 */  1662,  667,  370,  657,  571, 2176,  364, 2134,  362,  361,
- /*   950 */  1733,  539, 2281,  184, 1586, 1735, 1571, 2276,  617, 2084,
- /*   960 */  1520,  669,  653, 1892,  653, 1892, 2084, 2134, 2084, 1501,
- /*   970 */    91, 1275,  541,   42,   41,   40,  540, 2279,  254, 2084,
- /*   980 */   632,  669,  636, 2103, 1274,  703,  155,  154,  700,  699,
- /*   990 */   698,  152, 2115, 2098,  758, 2077, 2151,   47, 2084,  110,
- /*  1000 */  2117,  673, 2119, 2120,  668, 2116,  663, 2078,   14,   13,
- /*  1010 */  1732, 2295, 2115, 2204, 1731,  670, 2151,  387, 2200,  332,
- /*  1020 */  2117,  673, 2119, 2120,  668,  666,  663,  654, 2169, 2094,
- /*  1030 */  2100,   39,   38, 1578, 1579,   45,   43,   42,   41,   40,
- /*  1040 */   663, 2134,  591,  385, 1642, 2275,  485, 1989,  620,  653,
- /*  1050 */  1892,  165,  660, 2084,  591,  669,  664, 2275, 2084, 1894,
- /*  1060 */  2281,  184, 2084, 1551, 1561, 2276,  617,  291,  697, 1577,
- /*  1070 */  1580, 1935, 2281,  184, 2224, 1639,  396, 2276,  617, 1717,
- /*  1080 */  1718,  653, 1892, 1496,  165, 1494, 2115,  653, 1892, 1279,
- /*  1090 */  2151,  198, 1894,  170, 2117,  673, 2119, 2120,  668,  650,
- /*  1100 */   663,  576, 1278,  628,  140,  651, 1552,   52, 1619,    3,
- /*  1110 */   243,  701, 1499, 1500, 1935, 1550, 1553, 1554, 1555, 1556,
+ /*   680 */  1572, 1573, 1574, 1575,    2,   48,   46, 1581,  109,   70,
+ /*   690 */  1944, 2116,   69,  393,  619, 1495, 1744,  381,  653, 1892,
+ /*   700 */  1715,  631, 1552, 1221,  187, 1942, 1576, 1619, 1493,  187,
+ /*   710 */  2219,   34,  729,  727,  628,  140,  437,   41,   40,  653,
+ /*   720 */  1892,   47,   45,   44,   43,   42,  578, 2134,   81,   80,
+ /*   730 */   435,  541, 1571,  191,  164,  540, 2215,  446,  639, 2084,
+ /*   740 */  1222,  669,   41,   40, 2084, 1501,   47,   45,   44,   43,
+ /*   750 */    42,  653, 1892,  187,  339,  142,  697,  421, 2175, 1935,
+ /*   760 */   419,  415,  411,  408,  428,   12,  243,  653, 1892,  461,
+ /*   770 */   758, 1501, 2115,   49,  653, 1892, 2151, 2116,  639,  110,
+ /*   780 */  2117,  673, 2119, 2120,  668,  462,  663,  670,  637, 1767,
+ /*   790 */  2003,  181,  515, 2204,   36,  396,   87,  387, 2200, 1714,
+ /*   800 */    41,   40,  187,  165,   47,   45,   44,   43,   42, 1578,
+ /*   810 */  1579, 1894,  360, 2134,  193,  275, 2212,  627, 2231,  134,
+ /*   820 */   626, 1887, 2275,  628,  140, 2084,  696,  669,  648, 1944,
+ /*   830 */  2003,  653, 1892, 1468, 1469, 1685,  386,  615,  184, 1551,
+ /*   840 */  1561, 1989, 2276,  617, 1942, 1577, 1580,   41,   40, 1889,
+ /*   850 */   717,   47,   45,   44,   43,   42, 1867,  590, 2115, 1496,
+ /*   860 */  1275, 1494, 2151,  653, 1892,  110, 2117,  673, 2119, 2120,
+ /*   870 */   668, 1944,  663, 1274,  242, 1944, 1989, 2295,  241, 2204,
+ /*   880 */   363,  245,  397,  387, 2200,  194, 1943,  634, 1499, 1500,
+ /*   890 */  1942, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560,
+ /*   900 */   665,  661, 1569, 1570, 1572, 1573, 1574, 1575,    2,   48,
+ /*   910 */    46, 2116, 1743, 1174, 1175,  653, 1892,  393, 1742, 1495,
+ /*   920 */   198,  670, 1877, 2238,  183, 2212, 2213, 2067,  138, 2217,
+ /*   930 */  1576, 2116, 1493,  587,  591,  485, 1523, 2275,   90,  345,
+ /*   940 */  1662,  667,  370, 1643,  571, 1741,  364, 2134,  362,  361,
+ /*   950 */  1740,  539, 2281,  184, 1879,  569, 1571, 2276,  617, 2084,
+ /*   960 */  2084,  669,  653, 1892,  653, 1892, 2084, 2134,  567, 1501,
+ /*   970 */   565,  701,  541, 1875, 1935,  702,  540,   55, 1935, 2084,
+ /*   980 */   632,  669,  636, 2219, 1896,  703,  155,  154,  700,  699,
+ /*   990 */   698,  152, 2115, 2084,  758, 2077, 2151,   49, 2084,  110,
+ /*  1000 */  2117,  673, 2119, 2120,  668, 2116,  663, 2078,   54, 2214,
+ /*  1010 */     3, 2295, 2115, 2204, 1520,  670, 2151,  387, 2200,  332,
+ /*  1020 */  2117,  673, 2119, 2120,  668,  666,  663,  654, 2169,  653,
+ /*  1030 */  1892,   41,   40, 1578, 1579,   47,   45,   44,   43,   42,
+ /*  1040 */   399, 2134,  591, 1739,  249, 2275, 1586,  291,  165,  653,
+ /*  1050 */  1892,  153, 1520, 2084,  591,  669, 1894, 2275,  653, 1892,
+ /*  1060 */  2281,  184, 1279, 1551, 1561, 2276,  617,  650, 1607, 1577,
+ /*  1070 */  1580,  232, 2281,  184,  230, 1278,  651, 2276,  617, 2224,
+ /*  1080 */  1639,  653, 1892, 1496,  254, 1494, 2115, 1738,  653, 1892,
+ /*  1090 */  2151, 2084,  439,  170, 2117,  673, 2119, 2120,  668,  297,
+ /*  1100 */   663,  576, 1737,  628,  140,  440,  400, 1736,   56,  146,
+ /*  1110 */  1733,  135, 1499, 1500, 1732, 1550, 1553, 1554, 1555, 1556,
  /*  1120 */  1557, 1558, 1559, 1560,  665,  661, 1569, 1570, 1572, 1573,
- /*  1130 */  1574, 1575,    2,   46,   44,  653, 1892,  653, 1892, 1730,
- /*  1140 */  1729,  393,  399, 1495,  618, 2296, 1728, 2116,  591, 2070,
- /*  1150 */   165, 2275, 1727,  297, 1576,  400, 1493,  670, 1894, 2251,
- /*  1160 */   153,  623,  439, 1726,  487, 1725, 2281,  184, 1870,  252,
- /*  1170 */   310, 2276,  617, 1921, 2116,  440,  702, 1468, 1469, 1935,
- /*  1180 */  1571,   74,  232, 2134,  670,  230,  598, 2084, 2084,  146,
- /*  1190 */   573,  135,  572, 1501, 2084, 2084,  616,  669,  413, 2275,
- /*  1200 */  2084,  590, 1597,  544,  185, 2212, 2213, 1781,  138, 2217,
- /*  1210 */  2134, 2084,  148, 2084,  615,  184, 2244,   54,  758, 2276,
- /*  1220 */   617,   15, 2084,  234,  669, 1260,  233,  272, 2115,  556,
- /*  1230 */    82, 1504, 2151,  153, 2116,  110, 2117,  673, 2119, 2120,
- /*  1240 */   668, 1503,  663,  604,  670,  236,  619, 2295,  235, 2204,
- /*  1250 */  1774, 1772,  225,  387, 2200, 2115,  153, 1578, 1579, 2151,
- /*  1260 */    64,  711,  110, 2117,  673, 2119, 2120,  668,  238,  663,
- /*  1270 */  2134,  237,  558,  561, 2295,   64, 2204,  259,  621,  266,
- /*  1280 */   387, 2200, 2084, 1240,  669, 2105, 2135, 1551, 1561, 1998,
- /*  1290 */  1463, 1221,  404, 1577, 1580, 1759,  703,  155,  154,  700,
- /*  1300 */   699,  698,  152,   14,   13,  153, 1764, 1496,   48, 1494,
- /*  1310 */  1932,  284, 2234, 1466,   71, 2115,  151, 1671,  153, 2151,
- /*  1320 */   629,   53,  169, 2117,  673, 2119, 2120,  668, 1222,  663,
- /*  1330 */    48, 1768, 1670,   64,  261,   48, 1499, 1500, 2107, 1550,
+ /*  1130 */  1574, 1575,    2,   48,   46, 2084,  420,  487, 1870, 1731,
+ /*  1140 */  1730,  393,   35, 1495,  618, 2296, 1729, 2116,  591,  623,
+ /*  1150 */  2084, 2275, 1612, 1728, 1576, 2084, 1493,  670, 2084, 2251,
+ /*  1160 */  1552,  620, 2084, 1727, 1726, 1725, 2281,  184,   74,  543,
+ /*  1170 */   310, 2276,  617, 1921, 2116,  234,  236, 1504,  233,  235,
+ /*  1180 */  1571,  544,  148, 2134,  670, 1781,  598, 2084, 2084,  153,
+ /*  1190 */   252, 1262, 1642, 1501, 2084, 2084,  238,  669, 1552,  237,
+ /*  1200 */   573, 2084,  572, 1260,  185, 2212, 2213,  556,  138, 2217,
+ /*  1210 */  2134, 2084, 2084, 2084,  153,   50,   50,   82,  758, 1774,
+ /*  1220 */   259,   15, 2084, 1772,  669, 1717, 1718,  664, 2115, 1834,
+ /*  1230 */    14,   13, 2151,  153, 2116,  110, 2117,  673, 2119, 2120,
+ /*  1240 */   668,  558,  663, 1503,  670,  561, 1463, 2295,   50, 2204,
+ /*  1250 */  2105,  284,  660,  387, 2200, 2115,   71, 1578, 1579, 2151,
+ /*  1260 */   151,  153,  110, 2117,  673, 2119, 2120,  668, 2244,  663,
+ /*  1270 */  2134, 1466, 1671, 1670, 2295,   64, 2204,  261,   50, 1735,
+ /*  1280 */   387, 2200, 2084,  225,  669,  272,  604, 1551, 1561, 2135,
+ /*  1290 */   635,  266,  404, 1577, 1580, 1998,  703,  155,  154,  700,
+ /*  1300 */   699,  698,  152, 2107, 1759, 1423, 1932, 1496,  289, 1494,
+ /*  1310 */  2234,   50,  711,  645,  677, 2115,  151,  293, 1305, 2151,
+ /*  1320 */  1768,  153,  169, 2117,  673, 2119, 2120,  668, 1764,  663,
+ /*  1330 */  1507,  629, 1613,  136, 1240, 1562, 1499, 1500,  151, 1550,
  /*  1340 */  1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560,  665,  661,
- /*  1350 */  1569, 1570, 1572, 1573, 1574, 1575,    2,  271,  274,  390,
- /*  1360 */   389,  677,  635, 2242,  151, 1423,  153, 2116,  289, 1509,
- /*  1370 */   136,  645,  151,  293,  712, 1305,    1,  670,    5, 2269,
- /*  1380 */  1576,  753, 1502,  407, 1507,  412,  354,  309, 1446,  441,
- /*  1390 */  1613,  304, 1562,  624, 1506,  197, 1238, 2116, 1523, 1999,
- /*  1400 */   445,  478,  450, 2134, 1518,  463, 1571,  670, 1991, 2223,
- /*  1410 */   470,  477,  479,  488,  489, 2084,  202,  669, 1332, 1501,
- /*  1420 */   486, 1336,  201, 1343,  491,  492,  204, 1341,  494,  156,
- /*  1430 */   496, 1524,  497, 2134,    4,  498,  505,  506, 1526,  508,
- /*  1440 */   212, 1521,  509,  214,  659, 2084, 1525,  669, 2115,  510,
- /*  1450 */  1527,  511, 2151,  217,  513,  110, 2117,  673, 2119, 2120,
- /*  1460 */   668,  219,  663,   85,   86, 2116, 1194, 2295,  517, 2204,
- /*  1470 */   223,  536,  534,  387, 2200,  670,  535,  538, 2115,  344,
- /*  1480 */  2060, 1882, 2151, 2057,  229,  110, 2117,  673, 2119, 2120,
- /*  1490 */   668, 1878,  663,  112,  577,  575,  231, 2295,   89, 2204,
- /*  1500 */   158, 2134,  159,  387, 2200,  149, 1880, 1876,  160, 2056,
- /*  1510 */   246,  161,  582, 2084,  580,  669,  305,  581,  585,  250,
- /*  1520 */  1453,  588,    8,  605, 2250,  248,  643,  595, 2249,  586,
- /*  1530 */   257,  601,  614, 1510, 2235, 1505, 2245,  376,  608, 2226,
- /*  1540 */   265,  173,  596,  594,  260,  268, 2115,  267,  593,  622,
- /*  1550 */  2151, 2298,  625,  110, 2117,  673, 2119, 2120,  668,  269,
- /*  1560 */   663,  377, 1513, 1515, 2274, 2179,  139, 2204,  270, 1639,
- /*  1570 */  1522,  387, 2200, 2116, 2220,  661, 1569, 1570, 1572, 1573,
- /*  1580 */  1574, 1575,  279,  670,  633,  380, 1528,   96, 2004,  306,
- /*  1590 */   641,  642, 2018, 2116, 2017, 2016,  307,  646,  273,  383,
- /*  1600 */    98,  647,  308,  670,   61,  100, 1893, 2185,  102, 2134,
- /*  1610 */  1936,  311,  754, 1855, 2076,  755,  675,  757,   51,  346,
- /*  1620 */  2075, 2084,  347,  669, 2074,  315,  300,  335,  320, 2134,
- /*  1630 */   313,  334,   78, 2071,  409, 1486,  410, 1487,  324,  190,
- /*  1640 */   414, 2084,  416,  669, 2069,  417,  418, 2068,  355, 2066,
- /*  1650 */   422, 2065,  424, 2064, 2115,   79,  426, 1449, 2151, 1448,
- /*  1660 */  2030,  110, 2117,  673, 2119, 2120,  668, 2029,  663, 2028,
- /*  1670 */   433,  434, 2027, 2177, 2115, 2204, 1400, 2116, 2151,  387,
- /*  1680 */  2200,  110, 2117,  673, 2119, 2120,  668,  670,  663, 2026,
- /*  1690 */  1982, 1981, 1979,  656,  145, 2204, 1978, 1977, 1980,  387,
- /*  1700 */  2200, 2116, 1976, 1975, 1973, 1972, 1971,  195,  451, 1970,
- /*  1710 */   453,  670, 1984, 2134, 1969, 1968, 1967, 1966, 1965, 1964,
- /*  1720 */  1963, 1962, 1961, 1960, 1959, 2084, 1958,  669, 1957, 1956,
- /*  1730 */  1955, 1954, 1953, 1952, 1983,  147, 1951, 2134, 1950, 1949,
- /*  1740 */  1948, 1947,  481, 1946, 1945, 1797,  203, 1402,  342, 2084,
- /*  1750 */  1796,  669, 1795,  205,  206,  343, 1793, 1276, 2115, 1280,
- /*  1760 */  1754, 1176, 2151,  218, 2024,  111, 2117,  673, 2119, 2120,
- /*  1770 */   668,  178,  663, 1753, 2047, 2037, 2025, 1272, 2116, 2204,
- /*  1780 */  2002, 1871, 2115, 2203, 2200,   76, 2151,   77,  670,  111,
- /*  1790 */  2117,  673, 2119, 2120,  668,  208,  663,  216, 2104,  210,
- /*  1800 */  1792, 2116,  179, 2204,  503, 1790,  518,  658, 2200,  520,
- /*  1810 */   519,  670, 1788,  522, 2134, 1214,  523, 1786,  524,  526,
- /*  1820 */  1784,  527,  528,  530, 2116,  532, 2084, 1771,  669, 1770,
- /*  1830 */  1750,  531, 1873, 1348,  670, 1347, 1872, 2134, 1263, 1261,
- /*  1840 */   726, 1259, 1258, 1257, 1256, 1255,  728, 2116, 1250, 2084,
- /*  1850 */  1252,  669, 1782, 1251, 1249,  367, 1775,  670, 1773,  671,
- /*  1860 */  2134,  228,  368, 2151,  369,  559,  111, 2117,  673, 2119,
- /*  1870 */  2120,  668, 2084,  663,  669,   63,  562, 1749,  564, 1748,
- /*  1880 */  2204,  566, 2115, 2134,  349, 2200, 2151, 1747,  568,  111,
- /*  1890 */  2117,  673, 2119, 2120,  668, 2084,  663,  669,  113, 1473,
- /*  1900 */  1475,   27, 1472, 2204, 2046, 2115, 1459, 1455, 2201, 2151,
- /*  1910 */  2116,   67,  326, 2117,  673, 2119, 2120,  668, 1457,  663,
- /*  1920 */   670, 2036, 1477,  583, 2023, 2021, 2280,   20, 2115, 1687,
- /*  1930 */    56,   17, 2151,    6,   29,  170, 2117,  673, 2119, 2120,
- /*  1940 */   668,    7,  663,  589,  256,  584, 2134,  597,  258,  599,
- /*  1950 */    59,  382,   60,  373,  163,  613, 1669,  171, 2084,  251,
- /*  1960 */   669,  262,   30,  263, 1661,  264,   21,   65,   92, 2105,
- /*  1970 */    31, 1707, 1708,   22, 1702, 2116, 1701,  378, 1706, 1705,
- /*  1980 */   379, 1636, 1635, 2022,  276,  667, 2020, 2297, 2019, 2001,
- /*  1990 */    58, 2115,   94,   95,  174, 2151, 2116,  282,  333, 2117,
- /*  2000 */   673, 2119, 2120,  668,  283,  663,  670,   23,  644, 2116,
- /*  2010 */  1667, 2134,  285,  290,   68, 2000,   97,  292,  295,  670,
- /*  2020 */   103,   13,   24, 2084, 1511,  669, 1588,   99, 1587,   11,
- /*  2030 */  1543, 1598, 2134, 2154,  175, 1566, 1564,  392,  662,  188,
- /*  2040 */    57, 1563,  672,  674, 2084, 2134,  669,   18,   37,   16,
- /*  2050 */   394,   25,  676, 1535, 1333,   26, 2115, 2084,  395,  669,
- /*  2060 */  2151,  579,  678,  332, 2117,  673, 2119, 2120,  668, 1330,
- /*  2070 */   663,  680, 2170,  681,  683, 1327,  684, 2115,  686,  761,
- /*  2080 */  1321, 2151,  687,  689,  333, 2117,  673, 2119, 2120,  668,
- /*  2090 */  2115,  663, 1319,  303, 2151,  690, 2116,  333, 2117,  673,
- /*  2100 */  2119, 2120,  668, 1325,  663,  104,  670, 1324,  298,  176,
- /*  2110 */   105, 1342, 1323,   75, 1338,  751,  747,  743,  739,  301,
- /*  2120 */  2116, 1322, 1212,  704, 1246, 1245, 1244, 1270, 1243, 1242,
- /*  2130 */   670, 1241, 2134, 1239, 1237, 1236, 1235, 1233,  716,  299,
- /*  2140 */  1232, 1231, 1230, 1229, 2084, 1228,  669, 1227, 1265, 1267,
- /*  2150 */  1224, 1223, 1218, 1220, 1219, 1789, 2134, 1217,  736,  108,
- /*  2160 */   737, 1787,  294,  738,  740,  742, 1785,  744, 2084, 1783,
- /*  2170 */   669,  748, 1769,  746,  741,  752,  745,  574,  750, 1166,
- /*  2180 */   749, 2151, 1746, 2116,  328, 2117,  673, 2119, 2120,  668,
- /*  2190 */   302,  663,  756,  670,  649, 1497,  312,  759,  760, 1721,
- /*  2200 */  1721, 2115, 1721, 1721, 1721, 2151, 1721, 2116,  317, 2117,
- /*  2210 */   673, 2119, 2120,  668, 1721,  663, 1721,  670, 1721, 2134,
- /*  2220 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,  281,
- /*  2230 */  1721, 2084, 1721,  669,  280, 1721, 1721, 1721, 1721, 1721,
- /*  2240 */  1721, 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721,
- /*  2250 */  1721, 1721, 1721, 2116,  247, 2084, 1721,  669, 1721, 1721,
- /*  2260 */  1721, 1721, 1721,  670, 2115, 1721, 1721, 1721, 2151, 2116,
- /*  2270 */  1721,  318, 2117,  673, 2119, 2120,  668, 1721,  663,  670,
- /*  2280 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134,
- /*  2290 */  1721, 1721, 2151, 1721, 1721,  319, 2117,  673, 2119, 2120,
- /*  2300 */   668, 2084,  663,  669, 1721, 2134, 1721, 1721, 1721, 1721,
- /*  2310 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721,  669,
- /*  2320 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116,
- /*  2330 */  1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151,  670,
- /*  2340 */  1721,  325, 2117,  673, 2119, 2120,  668, 1721,  663, 1721,
- /*  2350 */  2115, 1721, 1721, 2116, 2151, 1721, 1721,  329, 2117,  673,
- /*  2360 */  2119, 2120,  668,  670,  663, 2134, 1721, 1721, 1721, 1721,
- /*  2370 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721,  669,
- /*  2380 */  2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2134,
- /*  2390 */   670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116,
- /*  2400 */  1721, 2084, 1721,  669, 1721, 1721, 1721, 1721, 1721,  670,
- /*  2410 */  2115, 1721, 1721, 1721, 2151, 1721, 2134,  321, 2117,  673,
- /*  2420 */  2119, 2120,  668, 1721,  663, 1721, 1721, 1721, 2084, 1721,
- /*  2430 */   669, 1721, 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721,
- /*  2440 */  1721,  330, 2117,  673, 2119, 2120,  668, 2084,  663,  669,
- /*  2450 */  2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
- /*  2460 */   670, 2115, 1721, 1721, 1721, 2151, 2116, 1721,  322, 2117,
- /*  2470 */   673, 2119, 2120,  668, 1721,  663,  670, 1721, 1721, 1721,
- /*  2480 */  2115, 1721, 1721, 1721, 2151, 1721, 2134,  331, 2117,  673,
- /*  2490 */  2119, 2120,  668, 1721,  663, 1721, 2116, 1721, 2084, 1721,
- /*  2500 */   669, 1721, 2134, 1721, 1721, 1721,  670, 1721, 1721, 1721,
- /*  2510 */  1721, 1721, 1721, 1721, 2084, 1721,  669, 2116, 1721, 1721,
- /*  2520 */  1721, 1721, 1721, 1721, 1721, 1721, 1721,  670, 1721, 1721,
- /*  2530 */  1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721,  323, 2117,
- /*  2540 */   673, 2119, 2120,  668, 2084,  663,  669, 2115, 1721, 1721,
- /*  2550 */  1721, 2151, 1721, 2134,  336, 2117,  673, 2119, 2120,  668,
- /*  2560 */  1721,  663, 1721, 2116, 1721, 2084, 1721,  669, 1721, 1721,
- /*  2570 */  1721, 1721, 1721,  670, 1721, 1721, 1721, 2115, 1721, 1721,
- /*  2580 */  1721, 2151, 1721, 1721,  337, 2117,  673, 2119, 2120,  668,
- /*  2590 */  1721,  663, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134,
- /*  2600 */  1721, 1721, 2151, 1721, 1721, 2128, 2117,  673, 2119, 2120,
- /*  2610 */   668, 2084,  663,  669, 1721, 1721, 1721, 1721, 1721, 1721,
- /*  2620 */  1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721,
- /*  2630 */  1721, 1721, 1721,  670, 1721, 1721, 2116, 1721, 1721, 1721,
- /*  2640 */  1721, 1721, 1721, 1721, 2115, 1721,  670, 1721, 2151, 1721,
- /*  2650 */  1721, 2127, 2117,  673, 2119, 2120,  668, 1721,  663, 2134,
- /*  2660 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
- /*  2670 */  1721, 2084, 2134,  669, 1721, 1721, 1721, 1721, 1721, 1721,
- /*  2680 */  1721, 1721, 1721, 1721, 2084, 1721,  669, 2116, 1721, 1721,
- /*  2690 */  1721, 1721, 1721, 1721, 1721, 1721, 1721,  670, 1721, 1721,
- /*  2700 */  1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 1721,
- /*  2710 */  1721, 2126, 2117,  673, 2119, 2120,  668, 2115,  663, 1721,
- /*  2720 */  1721, 2151, 1721, 2134,  351, 2117,  673, 2119, 2120,  668,
- /*  2730 */  1721,  663, 1721, 2116, 1721, 2084, 1721,  669, 1721, 1721,
- /*  2740 */  1721, 1721, 1721,  670, 1721, 1721, 1721, 1721, 2116, 1721,
- /*  2750 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,  670, 1721,
- /*  2760 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134,
- /*  2770 */  1721, 1721, 2151, 1721, 1721,  352, 2117,  673, 2119, 2120,
- /*  2780 */   668, 2084,  663,  669, 2134, 1721, 1721, 1721, 1721, 1721,
- /*  2790 */  1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721,  669, 1721,
- /*  2800 */  1721, 1721, 1721, 1721,  670, 1721, 1721, 1721, 1721, 2116,
- /*  2810 */  1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151,  670,
- /*  2820 */  1721,  348, 2117,  673, 2119, 2120,  668, 1721,  663, 2115,
- /*  2830 */  2134, 1721, 1721, 2151, 1721, 1721,  353, 2117,  673, 2119,
- /*  2840 */  2120,  668, 2084,  663,  669, 2134, 1721, 1721, 1721, 1721,
- /*  2850 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721,  669,
- /*  2860 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
- /*  2870 */  1721, 1721, 1721, 1721, 1721,  671, 1721, 1721, 1721, 2151,
- /*  2880 */  1721, 1721,  328, 2117,  673, 2119, 2120,  668, 1721,  663,
- /*  2890 */  2115, 1721, 1721, 1721, 2151, 1721, 1721,  327, 2117,  673,
- /*  2900 */  2119, 2120,  668, 1721,  663,
+ /*  1350 */  1569, 1570, 1572, 1573, 1574, 1575,    2,  274,  271,  390,
+ /*  1360 */   389,  712,    1, 2242,    5,  407,  412, 2116,  309, 1509,
+ /*  1370 */   753, 1332,  354, 1336, 1446,  304,  197,  670, 1343, 2269,
+ /*  1380 */  1576,  624, 1502, 1238,  441, 1523, 1999,  445,  450,  478,
+ /*  1390 */  1341,  621, 1518,  463, 1991,  156, 1506, 2116,  470,  477,
+ /*  1400 */   479,  488, 1597, 2134,  489,  486, 1571,  670,  201, 2223,
+ /*  1410 */   202,  491,  492, 1524,  497, 2084,  204,  669,  494, 1501,
+ /*  1420 */   496,    4,  498,  505,  506, 1526,  508,  212, 1521,  509,
+ /*  1430 */   214, 1525, 1527, 2134,  510,  513,  511,  217, 1194,  517,
+ /*  1440 */   112,  534,  535,  219,  659, 2084,   85,  669, 2115,   86,
+ /*  1450 */   223,  536, 2151,  538,  344,  110, 2117,  673, 2119, 2120,
+ /*  1460 */   668,  575,  663, 2060,   89, 2116,  577, 2295,  305, 2204,
+ /*  1470 */  1882,  229, 1878,  387, 2200,  670,  231,  158, 2115,  246,
+ /*  1480 */   159, 1880, 2151, 1876,  160,  110, 2117,  673, 2119, 2120,
+ /*  1490 */   668,  161,  663,  581, 2057,  580,  250, 2295, 1453, 2204,
+ /*  1500 */   588, 2134,  605,  387, 2200, 2056,  585,  595,  643,  601,
+ /*  1510 */  2250, 2235,  582, 2084, 2249,  669,  248,  149, 2245,    8,
+ /*  1520 */   376,  257,  608,  614,  586,  260,  596,  594, 2226,  593,
+ /*  1530 */   267,  265,  377, 1510,  625, 1505, 2298,  173,  622, 1639,
+ /*  1540 */   268,  139, 1522,  633,  380,  279, 2115,   96, 1528, 2004,
+ /*  1550 */  2151,  270,  646,  110, 2117,  673, 2119, 2120,  668,  306,
+ /*  1560 */   663,  641, 1513, 1515, 2220, 2179,  647, 2204,   98,  642,
+ /*  1570 */   307,  387, 2200, 2116, 2018,  661, 1569, 1570, 1572, 1573,
+ /*  1580 */  1574, 1575, 1893,  670,  100,   61,  269, 2017, 2185,  102,
+ /*  1590 */  2016,  383,  308, 2116,  675, 1936, 1855,  300,  311, 2274,
+ /*  1600 */   754,  757,  273,  670,  755,   53,  335,  320,  346, 2134,
+ /*  1610 */   347,  313,  315, 2076,  334,  324, 2075, 2074,   78, 2071,
+ /*  1620 */   409, 2084,  410,  669, 1486, 1487,  190,  414, 2069, 2134,
+ /*  1630 */   416,  417,  418, 2068,  355, 2066,  422, 2065, 2064,  426,
+ /*  1640 */   424, 2084,   79,  669, 1449, 1448, 2030, 2029, 2028,  433,
+ /*  1650 */   434, 2027, 2026, 1400, 2115, 1982, 1981, 1979, 2151, 1978,
+ /*  1660 */  1977,  110, 2117,  673, 2119, 2120,  668,  145,  663, 1980,
+ /*  1670 */  1976, 1975, 1973, 2177, 2115, 2204,  195, 2116, 2151,  387,
+ /*  1680 */  2200,  110, 2117,  673, 2119, 2120,  668,  670,  663, 1972,
+ /*  1690 */  1971, 1970,  453,  656, 1984, 2204,  451, 1969, 1968,  387,
+ /*  1700 */  2200, 2116, 1967, 1966, 1965, 1964, 1963, 1962, 1961, 1960,
+ /*  1710 */  1959,  670, 1958, 2134, 1957, 1956, 1955,  147, 1954, 1953,
+ /*  1720 */  1952, 1983, 1951, 1950, 1949, 2084, 1948,  669, 1947, 1946,
+ /*  1730 */  1945,  481,  342, 1402,  343, 1276, 1797, 2134,  203, 1280,
+ /*  1740 */  1796,  205, 1795, 1793,  206, 1754, 2104,  179, 1176, 2084,
+ /*  1750 */   218,  669, 1753, 2047, 2037, 2025, 2024, 2002, 2115, 1871,
+ /*  1760 */  1272, 1214, 2151, 1792, 1790,  111, 2117,  673, 2119, 2120,
+ /*  1770 */   668,   76,  663,  208,  503,  210,   77,  178, 2116, 2204,
+ /*  1780 */   216,  518, 2115, 2203, 2200, 1788, 2151,  519,  670,  111,
+ /*  1790 */  2117,  673, 2119, 2120,  668,  522,  663,  520,  523, 1786,
+ /*  1800 */   524, 2116,  526, 2204,  528, 1784,  530,  658, 2200, 1771,
+ /*  1810 */  1770,  670,  532, 1750, 2134,  527, 1873,  531, 1348, 1347,
+ /*  1820 */  1872, 1263, 1261, 1259, 2116, 1258, 2084, 1257,  669, 1256,
+ /*  1830 */  1255, 1252, 1782,  726,  670,  728, 1251, 2134, 1249, 1250,
+ /*  1840 */   367, 1775, 1773,  368,  559,  369,  562, 2116,   63, 2084,
+ /*  1850 */  1749,  669, 1748, 1747,  228,  568,  113,  670, 2046,  671,
+ /*  1860 */  2134,  564,  566, 2151, 1473,  374,  111, 2117,  673, 2119,
+ /*  1870 */  2120,  668, 2084,  663,  669, 1475, 1477,   29, 1472,   67,
+ /*  1880 */  2204, 1459, 2115, 2134,  349, 2200, 2151, 1457,  375,  111,
+ /*  1890 */  2117,  673, 2119, 2120,  668, 2084,  663,  669, 1455, 2036,
+ /*  1900 */   583, 2023, 2021, 2204,   20, 2115, 2280,   31, 2201, 2151,
+ /*  1910 */  2116,  599,  333, 2117,  673, 2119, 2120,  668,  251,  663,
+ /*  1920 */   670,   17,    6, 1687,  256,    7,  589,  264, 2115,  597,
+ /*  1930 */    58,  258, 2151,  163,   21,  333, 2117,  673, 2119, 2120,
+ /*  1940 */   668,   22,  663, 2105,  584, 1669, 2134,  171,  373,  262,
+ /*  1950 */   263,   32,   33,   65,   24, 1661,   92,   23, 2084, 1702,
+ /*  1960 */   669, 1701, 1707, 1708,  378, 1706, 1705,  379,  276, 1636,
+ /*  1970 */  1635,   60,  174, 2022, 2020, 2116, 2019, 2001,   95,   94,
+ /*  1980 */   282,   25, 2000,  283,  285,  670, 1667,  290,   68,   99,
+ /*  1990 */   644, 2115,   97,  295,  292, 2151, 2116,   13,  326, 2117,
+ /*  2000 */   673, 2119, 2120,  668,  103,  663,  670,   26, 1511, 2116,
+ /*  2010 */  1588, 2134, 1587, 1598,  175,   11, 2154,  188, 1566,  667,
+ /*  2020 */   662, 1543,   39, 2084,   59,  669,  674,  676,   18, 1564,
+ /*  2030 */  1338,  672, 2134,  395,  298, 1563,   16,  382,   27,  680,
+ /*  2040 */    28,  613, 1535, 1333, 2084, 2134,  669,  678,  683, 1330,
+ /*  2050 */  1327,  681,  684,  686,  689, 1321, 2115, 2084,  687,  669,
+ /*  2060 */  2151, 1319,  690,  170, 2117,  673, 2119, 2120,  668,  104,
+ /*  2070 */   663,  105, 1342, 1246,  704,   75, 1212, 2115, 1325, 1324,
+ /*  2080 */  1245, 2151, 1244, 2116,  333, 2117,  673, 2119, 2120,  668,
+ /*  2090 */  2115,  663, 1243,  670, 2151, 1242, 1241,  332, 2117,  673,
+ /*  2100 */  2119, 2120,  668, 2116,  663, 1323, 2170, 1322, 1239, 1237,
+ /*  2110 */  1236, 1235, 1270,  670,  299, 2297,  716, 1233, 1232, 2134,
+ /*  2120 */  1231, 1230, 1229, 1228,  392, 1227, 1267, 1265, 1224, 1223,
+ /*  2130 */  1220, 2084, 1219,  669, 1218, 1217, 1789,  736,  737, 2134,
+ /*  2140 */   738, 1787,  740,  742,  394,  741, 1785,  745,  744,  746,
+ /*  2150 */  1783, 2084,  748,  669,  749,  750, 1769,  752, 1166, 1746,
+ /*  2160 */   302,  756, 1721,  759, 2115, 1497,  312,  760, 2151, 1721,
+ /*  2170 */  1721,  333, 2117,  673, 2119, 2120,  668, 1721,  663, 1721,
+ /*  2180 */  1721, 1721, 1721, 1721, 2115, 1721,  579, 1721, 2151, 1721,
+ /*  2190 */  1721,  333, 2117,  673, 2119, 2120,  668, 1721,  663, 1721,
+ /*  2200 */  2116, 1721, 1721, 1721,  761, 1721, 1721, 1721, 1721, 1721,
+ /*  2210 */   670, 1721, 1721, 1721, 1721, 1721, 2116, 1721,  303, 1721,
+ /*  2220 */  1721, 1721, 1721, 1721, 1721, 1721,  670, 1721, 1721, 1721,
+ /*  2230 */  1721, 1721, 1721, 1721,  176, 1721, 2134, 1721, 1721, 1721,
+ /*  2240 */   751,  747,  743,  739,  301, 1721, 2116, 1721, 2084, 1721,
+ /*  2250 */   669, 1721, 2134, 1721, 1721, 1721,  670, 1721, 1721, 1721,
+ /*  2260 */  1721, 1721, 1721, 1721, 2084, 1721,  669, 1721, 1721, 1721,
+ /*  2270 */  1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721,
+ /*  2280 */  1721,  574, 2134, 1721,  108, 2151,  670,  294,  328, 2117,
+ /*  2290 */   673, 2119, 2120,  668, 2084,  663,  669, 2115, 1721, 1721,
+ /*  2300 */  1721, 2151, 1721, 1721,  317, 2117,  673, 2119, 2120,  668,
+ /*  2310 */  1721,  663, 2134, 1721, 1721, 1721, 1721, 1721, 1721,  649,
+ /*  2320 */  1721, 1721, 1721, 1721, 2084, 1721,  669, 2115, 1721, 1721,
+ /*  2330 */  1721, 2151, 1721, 1721,  318, 2117,  673, 2119, 2120,  668,
+ /*  2340 */  2116,  663, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2350 */   670, 1721, 1721, 2116,  281, 1721, 1721, 2115, 1721,  280,
+ /*  2360 */  1721, 2151, 1721,  670,  319, 2117,  673, 2119, 2120,  668,
+ /*  2370 */  1721,  663, 1721, 1721, 1721, 1721, 2134, 1721, 1721,  247,
+ /*  2380 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134,
+ /*  2390 */   669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2400 */  1721, 2084, 1721,  669, 2116, 1721, 1721, 1721, 1721, 1721,
+ /*  2410 */  1721, 1721, 1721, 1721,  670, 1721, 1721, 2116, 1721, 1721,
+ /*  2420 */  1721, 2115, 1721, 1721, 1721, 2151, 1721,  670,  325, 2117,
+ /*  2430 */   673, 2119, 2120,  668, 2115,  663, 1721, 1721, 2151, 1721,
+ /*  2440 */  2134,  329, 2117,  673, 2119, 2120,  668, 1721,  663, 1721,
+ /*  2450 */  1721, 1721, 2084, 2134,  669, 1721, 1721, 1721, 1721, 1721,
+ /*  2460 */  1721, 1721, 1721, 1721, 1721, 2084, 1721,  669, 1721, 1721,
+ /*  2470 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721,
+ /*  2480 */  1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721,  670, 2151,
+ /*  2490 */  1721, 1721,  321, 2117,  673, 2119, 2120,  668, 2115,  663,
+ /*  2500 */  1721, 1721, 2151, 1721, 1721,  330, 2117,  673, 2119, 2120,
+ /*  2510 */   668, 1721,  663, 1721, 2134, 1721, 1721, 1721, 1721, 1721,
+ /*  2520 */  1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721,  669, 2116,
+ /*  2530 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,  670,
+ /*  2540 */  1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721,
+ /*  2550 */  1721, 1721, 1721, 1721,  670, 1721, 1721, 1721, 1721, 2115,
+ /*  2560 */  1721, 1721, 1721, 2151, 1721, 2134,  322, 2117,  673, 2119,
+ /*  2570 */  2120,  668, 1721,  663, 1721, 1721, 1721, 2084, 1721,  669,
+ /*  2580 */  2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2590 */  2116, 1721, 2084, 1721,  669, 1721, 1721, 1721, 1721, 1721,
+ /*  2600 */   670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2610 */  2115, 1721, 1721, 1721, 2151, 1721, 1721,  331, 2117,  673,
+ /*  2620 */  2119, 2120,  668, 1721,  663, 2115, 2134, 1721, 1721, 2151,
+ /*  2630 */  1721, 1721,  323, 2117,  673, 2119, 2120,  668, 2084,  663,
+ /*  2640 */   669, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2650 */  1721,  670, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721,
+ /*  2660 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,  670, 1721,
+ /*  2670 */  1721, 2115, 1721, 1721, 1721, 2151, 1721, 2134,  336, 2117,
+ /*  2680 */   673, 2119, 2120,  668, 1721,  663, 1721, 2116, 1721, 2084,
+ /*  2690 */  1721,  669, 1721, 1721, 2134, 1721, 1721,  670, 1721, 1721,
+ /*  2700 */  1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721,  669, 1721,
+ /*  2710 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2720 */  1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721,  337,
+ /*  2730 */  2117,  673, 2119, 2120,  668, 2084,  663,  669, 1721, 2115,
+ /*  2740 */  1721, 1721, 1721, 2151, 1721, 1721, 2128, 2117,  673, 2119,
+ /*  2750 */  2120,  668, 1721,  663, 1721, 2116, 1721, 1721, 1721, 1721,
+ /*  2760 */  1721, 1721, 1721, 1721, 1721,  670, 1721, 1721, 2115, 1721,
+ /*  2770 */  1721, 1721, 2151, 1721, 2116, 2127, 2117,  673, 2119, 2120,
+ /*  2780 */   668, 1721,  663, 1721,  670, 1721, 1721, 2116, 1721, 1721,
+ /*  2790 */  1721, 2134, 1721, 1721, 1721, 1721, 1721,  670, 1721, 1721,
+ /*  2800 */  1721, 1721, 1721, 2084, 1721,  669, 1721, 1721, 1721, 1721,
+ /*  2810 */  2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2820 */  1721, 1721, 2084, 2134,  669, 1721, 1721, 1721, 1721, 1721,
+ /*  2830 */  1721, 1721, 1721, 1721, 1721, 2084, 2115,  669, 1721, 1721,
+ /*  2840 */  2151, 1721, 1721, 2126, 2117,  673, 2119, 2120,  668, 1721,
+ /*  2850 */   663, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151,
+ /*  2860 */  2116, 1721,  351, 2117,  673, 2119, 2120,  668, 2115,  663,
+ /*  2870 */   670, 1721, 2151, 2116, 1721,  352, 2117,  673, 2119, 2120,
+ /*  2880 */   668, 1721,  663,  670, 1721, 1721, 1721, 1721, 2116, 1721,
+ /*  2890 */  1721, 1721, 1721, 1721, 1721, 1721, 2134, 1721,  670, 1721,
+ /*  2900 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134,
+ /*  2910 */   669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  2920 */  1721, 2084, 1721,  669, 2134, 1721, 1721, 1721, 1721, 1721,
+ /*  2930 */  1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721,  669, 1721,
+ /*  2940 */  1721, 2115, 1721, 1721,  670, 2151, 1721, 1721,  348, 2117,
+ /*  2950 */   673, 2119, 2120,  668, 2115,  663, 1721, 1721, 2151, 1721,
+ /*  2960 */  1721,  353, 2117,  673, 2119, 2120,  668, 1721,  663,  671,
+ /*  2970 */  2134, 1721, 1721, 2151, 1721, 1721,  328, 2117,  673, 2119,
+ /*  2980 */  2120,  668, 2084,  663,  669, 1721, 1721, 1721, 1721, 1721,
+ /*  2990 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  3000 */  1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721,
+ /*  3010 */  1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151,
+ /*  3020 */  1721, 1721,  327, 2117,  673, 2119, 2120,  668, 1721,  663,
 };
 static const YYCODETYPE yy_lookahead[] = {
- /*     0 */   333,  370,  383,  337,  400,  370,  340,  341,  404,  371,
- /*    10 */   343,  365,   12,   13,   14,    0,  371,  398,  399,  381,
- /*    20 */    20,   14,   22,   20,    8,    9,  381,   20,   12,   13,
- /*    30 */    14,   15,   16,   33,    0,   35,  369,  383,   20,   24,
- /*    40 */    25,   26,   27,   28,   29,   30,   31,   32,  381,  395,
- /*    50 */   383,  447,  398,  399,  450,  417,  418,  419,  412,   59,
- /*    60 */    44,  369,  417,  418,  419,   65,  428,   20,  376,  465,
- /*    70 */   466,   20,   72,  428,  470,  471,  384,  342,  343,    0,
+ /*     0 */   333,  338,  383,  370,  400,  342,  371,  344,  404,  371,
+ /*    10 */   343,  383,   12,   13,   14,    0,  381,  398,  399,  381,
+ /*    20 */    20,   20,   22,  395,    8,    9,  398,  399,   12,   13,
+ /*    30 */    14,   15,   16,   33,    0,   35,  369,  368,   20,   24,
+ /*    40 */    25,   26,   27,   28,   29,   30,   31,   32,  381,  380,
+ /*    50 */   383,  447,  417,  418,  450,  417,  418,  419,  379,   59,
+ /*    60 */    44,  382,  383,  428,  337,   65,  428,  340,  341,  465,
+ /*    70 */   466,  332,   72,  334,  470,  471,   20,  342,  343,    0,
  /*    80 */   337,  414,  348,  340,  341,  418,  436,  437,  421,  422,
- /*    90 */   423,  424,  425,  426,  332,  428,  334,   97,  364,   39,
+ /*    90 */   423,  424,  425,  426,  432,  428,  434,   97,  364,   39,
  /*   100 */   100,   67,   68,   69,   70,   71,  372,   73,   74,   75,
  /*   110 */    76,   77,   78,   79,   80,   81,   82,   83,   84,   85,
  /*   120 */    86,   87,   88,   89,   90,   91,   92,   93,  461,  462,
- /*   130 */   379,  420,   97,  382,  383,   21,  136,  137,   24,   25,
+ /*   130 */   100,  420,   97,  382,  383,   21,  136,  137,   24,   25,
  /*   140 */    26,   27,   28,   29,   30,   31,   32,  112,  113,  114,
  /*   150 */   115,  116,  117,  118,  119,  120,  121,  446,  123,  124,
- /*   160 */   125,  126,  127,  128,  369,   20,  166,  167,  447,  166,
- /*   170 */   167,  450,  172,  173,  369,  342,  343,  442,  443,  444,
- /*   180 */   445,  376,  447,  448,  389,  390,  186,  466,  188,  384,
- /*   190 */   330,  470,  471,  360,  130,  131,   64,    8,    9,  135,
+ /*   160 */   125,  126,  127,  128,   14,   20,  166,  167,  447,  168,
+ /*   170 */    20,  450,  172,  173,  369,  342,  343,  442,  443,  444,
+ /*   180 */   445,  376,  447,  448,  166,  167,  186,  466,  188,  384,
+ /*   190 */   330,  470,  471,  360,  130,  131,  343,    8,    9,  135,
  /*   200 */   367,   12,   13,   14,   15,   16,   59,  100,  129,  130,
  /*   210 */   131,  132,  133,  134,  135,  215,  216,    0,  218,  219,
  /*   220 */   220,  221,  222,  223,  224,  225,  226,  227,  228,  229,
  /*   230 */   230,  231,  232,  233,  234,  235,   12,   13,  181,    4,
- /*   240 */    20,   18,  100,   20,   20,  100,   22,  100,   20,  102,
+ /*   240 */   387,   18,  100,   20,   20,  100,   22,  100,  350,  102,
  /*   250 */    27,  333,  447,   30,   65,  450,   33,   33,  369,   35,
  /*   260 */   400,  343,  205,  206,  404,   12,   13,   14,   15,   16,
- /*   270 */   465,  466,   49,  100,   51,  470,  471,   54,   43,  390,
- /*   280 */    45,   46,  368,   59,   67,   68,   69,  369,    0,   65,
- /*   290 */   100,   74,   75,   76,  380,   20,   72,   80,  109,  381,
+ /*   270 */   465,  466,   49,  375,   51,  470,  471,   54,   43,  390,
+ /*   280 */    45,   46,  252,   59,   67,   68,   69,  369,    0,   65,
+ /*   290 */   100,   74,   75,   76,  355,  356,   72,   80,  109,  381,
  /*   300 */   110,  383,   85,   86,   87,   88,   20,  447,   91,   21,
  /*   310 */   450,  251,   24,   25,   26,   27,   28,   29,   30,   31,
  /*   320 */    32,   97,   99,  348,  100,  465,  466,  185,    3,  187,
@@ -550,378 +562,390 @@ static const YYCODETYPE yy_lookahead[] = {
  /*   350 */   342,  433,  434,  435,  165,  360,  214,  439,  440,  252,
  /*   360 */   136,  137,  367,  140,   20,   52,  143,  144,  145,  146,
  /*   370 */   147,  148,  149,  150,  151,  152,  153,  154,  155,  156,
- /*   380 */   157,  158,  159,  338,  161,  162,  163,  342,  168,  344,
- /*   390 */   166,  167,  447,  385,  252,  450,  172,  173,   14,  252,
- /*   400 */    67,   68,   69,   64,   20,  382,  383,   74,   75,   76,
+ /*   380 */   157,  158,  159,  369,  161,  162,  163,  342,  343,   20,
+ /*   390 */   166,  167,  447,  385,  252,  450,  172,  173,  369,  252,
+ /*   400 */    67,   68,   69,  389,  390,  360,  377,   74,   75,   76,
  /*   410 */   186,  466,  188,   80,  171,  470,  471,    0,   85,   86,
- /*   420 */    87,   88,  136,  137,   91,  252,  101,   35,  239,  240,
+ /*   420 */    87,   88,  136,  137,   91,    3,  101,   35,  239,  240,
  /*   430 */   241,  242,  243,  244,  245,  246,  247,  248,  249,  215,
- /*   440 */   216,  166,  218,  219,  220,  221,  222,  223,  224,  225,
+ /*   440 */   216,    0,  218,  219,  220,  221,  222,  223,  224,  225,
  /*   450 */   226,  227,  228,  229,  230,  231,  232,  233,  234,  235,
- /*   460 */   236,   12,   13,   33,   72,   22,  351,   20,  338,   20,
- /*   470 */   333,   22,  342,  358,  344,  215,  111,   20,   35,   49,
- /*   480 */   343,   64,   33,   81,   35,   55,   56,   57,   58,   59,
- /*   490 */   420,    8,    9,  168,  333,   12,   13,   14,   15,   16,
- /*   500 */   257,  258,  259,   20,   20,    4,  369,  371,   59,  355,
- /*   510 */   356,   72,   20,  343,   65,   72,  446,  381,  381,  342,
+ /*   460 */   236,   12,   13,   33,   72,   22,   49,   20,  343,   20,
+ /*   470 */   333,   22,   20,   59,  351,  215,   64,   20,   35,   49,
+ /*   480 */   343,  358,   33,   81,   35,   55,   56,   57,   58,   59,
+ /*   490 */    59,    8,    9,  168,  369,   12,   13,   14,   15,   16,
+ /*   500 */   257,  258,  259,   20,   20,    4,  369,  371,   59,  236,
+ /*   510 */    20,  238,   22,   99,   65,   72,  102,  381,  381,  342,
  /*   520 */   383,   72,  262,  263,  264,  265,  266,  267,  268,   99,
- /*   530 */     8,    9,  102,  236,   12,   13,   14,   15,   16,  369,
- /*   540 */    97,  369,  381,  141,  142,   13,   97,  333,  376,  100,
- /*   550 */   166,  414,  165,  417,  418,  418,  384,  100,  421,  422,
- /*   560 */   423,  424,  425,  426,  428,  428,  164,   35,   18,  392,
+ /*   530 */     8,    9,  102,  102,   12,   13,   14,   15,   16,    0,
+ /*   540 */    97,  338,   52,  141,  142,  342,   97,  344,  423,  100,
+ /*   550 */    14,  414,  100,  417,  418,  418,   20,  100,  421,  422,
+ /*   560 */   423,  424,  425,  426,  428,  428,  164,  365,   18,  392,
  /*   570 */   433,  394,  435,   23,    8,    9,  439,  440,   12,   13,
  /*   580 */    14,   15,   16,  136,  137,  136,  137,   37,   38,  452,
- /*   590 */   333,   41,  100,  423,    4,  381,   20,  460,   22,  169,
- /*   600 */   170,  100,  350,  342,  174,  236,  176,  238,  420,   19,
- /*   610 */    60,   61,   62,   63,  111,  166,  167,  361,  366,  172,
- /*   620 */   173,  172,  173,   33,  194,  369,  239,  375,   52,  186,
- /*   630 */    72,  188,  376,  377,  446,  186,  249,  188,  381,   49,
- /*   640 */   384,    0,  342,  343,   54,  342,  343,  369,  333,   59,
- /*   650 */   100,  168,  168,  392,  376,  394,  371,  129,  215,  216,
- /*   660 */   360,  133,  384,  360,  215,  216,  381,  218,  219,  220,
+ /*   590 */   432,   41,  434,  357,    4,  359,  155,  460,  333,  169,
+ /*   600 */   170,  100,   64,   64,  174,  164,  176,    1,    2,   19,
+ /*   610 */    60,   61,   62,   63,  412,  166,  167,  361,  361,  172,
+ /*   620 */   173,  172,  173,   33,  194,  369,  369,  342,  343,  186,
+ /*   630 */    14,  188,  376,  377,  377,  186,   20,  188,  447,   49,
+ /*   640 */   384,  450,  342,  343,   54,  360,  381,   20,  369,   59,
+ /*   650 */   100,  168,   14,   15,   16,  376,  465,  466,  215,  216,
+ /*   660 */   360,  470,  471,  384,  215,  216,   20,  218,  219,  220,
  /*   670 */   221,  222,  223,  224,  225,  226,  227,  228,  229,  230,
  /*   680 */   231,  232,  233,  234,  235,   12,   13,   14,  138,   99,
- /*   690 */   369,  333,  102,   20,  370,   22,  381,  376,  342,  343,
- /*   700 */   178,  343,  417,  418,  419,  384,   33,  333,   35,  252,
- /*   710 */   357,    2,  359,  428,  342,  343,  360,    8,    9,  342,
- /*   720 */   343,   12,   13,   14,   15,   16,  358,  369,  178,  179,
- /*   730 */   180,  350,   59,  183,  168,  355,  356,  360,  342,  381,
- /*   740 */   370,  383,    8,    9,  252,   72,   12,   13,   14,   15,
- /*   750 */    16,  342,  343,  252,  204,  381,  375,  207,    0,  343,
- /*   760 */   210,  211,  212,  213,  214,   14,  333,  342,  343,  360,
- /*   770 */    97,   20,  414,  100,  342,  343,  418,  333,  333,  421,
+ /*   690 */   369,  333,  102,   20,  272,   22,  333,  376,  342,  343,
+ /*   700 */   178,  343,  166,   35,  252,  384,   33,  101,   35,  252,
+ /*   710 */   420,    2,  355,  356,  342,  343,  360,    8,    9,  342,
+ /*   720 */   343,   12,   13,   14,   15,   16,  111,  369,  178,  179,
+ /*   730 */   180,  129,   59,  183,  168,  133,  446,  360,  342,  381,
+ /*   740 */    72,  383,    8,    9,  381,   72,   12,   13,   14,   15,
+ /*   750 */    16,  342,  343,  252,  204,  431,  378,  207,  434,  381,
+ /*   760 */   210,  211,  212,  213,  214,  236,  130,  342,  343,  360,
+ /*   770 */    97,   72,  414,  100,  342,  343,  418,  333,  342,  421,
  /*   780 */   422,  423,  424,  425,  426,  360,  428,  343,  392,  345,
- /*   790 */   394,  433,  360,  435,    2,    4,  333,  439,  440,  277,
- /*   800 */     8,    9,  252,  387,   12,   13,   14,   15,   16,  136,
- /*   810 */   137,  342,  343,  369,  381,  443,  444,  445,  460,  447,
- /*   820 */   448,   21,  450,  342,  343,  381,  381,  383,  370,  360,
- /*   830 */    20,  342,  343,   59,   34,  101,   36,  465,  466,  166,
- /*   840 */   167,  343,  470,  471,  381,  172,  173,    8,    9,  360,
- /*   850 */   209,   12,   13,   14,   15,   16,    0,  431,  414,  186,
- /*   860 */   434,  188,  418,  342,  343,  421,  422,  423,  424,  425,
- /*   870 */   426,  369,  428,   99,  131,  370,  102,  433,  135,  435,
- /*   880 */    37,  360,  369,  439,  440,  387,  384,  400,  215,  216,
- /*   890 */   377,  218,  219,  220,  221,  222,  223,  224,  225,  226,
+ /*   790 */   394,  433,  360,  435,    2,  361,  350,  439,  440,  277,
+ /*   800 */     8,    9,  252,  369,   12,   13,   14,   15,   16,  136,
+ /*   810 */   137,  377,  366,  369,  168,  443,  444,  445,  460,  447,
+ /*   820 */   448,  375,  450,  342,  343,  381,  111,  383,  392,  369,
+ /*   830 */   394,  342,  343,  197,  198,  101,  376,  465,  466,  166,
+ /*   840 */   167,  343,  470,  471,  384,  172,  173,    8,    9,  360,
+ /*   850 */    72,   12,   13,   14,   15,   16,    0,   48,  414,  186,
+ /*   860 */    22,  188,  418,  342,  343,  421,  422,  423,  424,  425,
+ /*   870 */   426,  369,  428,   35,  131,  369,  343,  433,  135,  435,
+ /*   880 */    37,  360,  376,  439,  440,  387,  384,  400,  215,  216,
+ /*   890 */   384,  218,  219,  220,  221,  222,  223,  224,  225,  226,
  /*   900 */   227,  228,  229,  230,  231,  232,  233,  234,  235,   12,
- /*   910 */    13,  333,  432,  155,  434,  342,  343,   20,  333,   22,
- /*   920 */   333,  343,  164,  345,  443,  444,  445,   59,  447,  448,
- /*   930 */    33,  333,   35,  360,  447,   45,   46,  450,  195,  196,
- /*   940 */   101,  343,  199,  432,  201,  434,  103,  369,  105,  106,
- /*   950 */   333,  108,  465,  466,   14,  334,   59,  470,  471,  381,
- /*   960 */    20,  383,  342,  343,  342,  343,  381,  369,  381,   72,
- /*   970 */   102,   22,  129,   14,   15,   16,  133,    3,  168,  381,
- /*   980 */   360,  383,  360,  371,   35,  129,  130,  131,  132,  133,
+ /*   910 */    13,  333,  333,   45,   46,  342,  343,   20,  333,   22,
+ /*   920 */   387,  343,  370,  345,  443,  444,  445,    0,  447,  448,
+ /*   930 */    33,  333,   35,  360,  447,   97,   20,  450,  195,  196,
+ /*   940 */   101,  343,  199,    4,  201,  333,  103,  369,  105,  106,
+ /*   950 */   333,  108,  465,  466,  370,   21,   59,  470,  471,  381,
+ /*   960 */   381,  383,  342,  343,  342,  343,  381,  369,   34,   72,
+ /*   970 */    36,  378,  129,  370,  381,  378,  133,  168,  381,  381,
+ /*   980 */   360,  383,  360,  420,  370,  129,  130,  131,  132,  133,
  /*   990 */   134,  135,  414,  381,   97,  400,  418,  100,  381,  421,
- /*  1000 */   422,  423,  424,  425,  426,  333,  428,  400,    1,    2,
- /*  1010 */   333,  433,  414,  435,  333,  343,  418,  439,  440,  421,
- /*  1020 */   422,  423,  424,  425,  426,  427,  428,  429,  430,  417,
- /*  1030 */   418,    8,    9,  136,  137,   12,   13,   14,   15,   16,
- /*  1040 */   428,  369,  447,  361,  253,  450,   97,  343,   44,  342,
- /*  1050 */   343,  369,   65,  381,  447,  383,  370,  450,  381,  377,
- /*  1060 */   465,  466,  381,  166,  167,  470,  471,  360,  378,  172,
- /*  1070 */   173,  381,  465,  466,  250,  251,  361,  470,  471,  136,
- /*  1080 */   137,  342,  343,  186,  369,  188,  414,  342,  343,   22,
- /*  1090 */   418,  387,  377,  421,  422,  423,  424,  425,  426,  360,
- /*  1100 */   428,  400,   35,  342,  343,  360,  166,   42,  101,   44,
- /*  1110 */   130,  378,  215,  216,  381,  218,  219,  220,  221,  222,
+ /*  1000 */   422,  423,  424,  425,  426,  333,  428,  400,   42,  446,
+ /*  1010 */    44,  433,  414,  435,   20,  343,  418,  439,  440,  421,
+ /*  1020 */   422,  423,  424,  425,  426,  427,  428,  429,  430,  342,
+ /*  1030 */   343,    8,    9,  136,  137,   12,   13,   14,   15,   16,
+ /*  1040 */   361,  369,  447,  333,  370,  450,   14,  360,  369,  342,
+ /*  1050 */   343,   44,   20,  381,  447,  383,  377,  450,  342,  343,
+ /*  1060 */   465,  466,   22,  166,  167,  470,  471,  360,  165,  172,
+ /*  1070 */   173,  104,  465,  466,  107,   35,  360,  470,  471,  250,
+ /*  1080 */   251,  342,  343,  186,  168,  188,  414,  333,  342,  343,
+ /*  1090 */   418,  381,   22,  421,  422,  423,  424,  425,  426,  360,
+ /*  1100 */   428,  400,  333,  342,  343,   35,  360,  333,  101,   42,
+ /*  1110 */   333,   44,  215,  216,  333,  218,  219,  220,  221,  222,
  /*  1120 */   223,  224,  225,  226,  227,  228,  229,  230,  231,  232,
- /*  1130 */   233,  234,  235,   12,   13,  342,  343,  342,  343,  333,
- /*  1140 */   333,   20,  361,   22,  472,  473,  333,  333,  447,    0,
- /*  1150 */   369,  450,  333,  360,   33,  360,   35,  343,  377,  345,
- /*  1160 */    44,   44,   22,  333,   97,  333,  465,  466,    0,  405,
- /*  1170 */   362,  470,  471,  365,  333,   35,  378,  197,  198,  381,
- /*  1180 */    59,  111,  104,  369,  343,  107,  345,  381,  381,   42,
- /*  1190 */   200,   44,  202,   72,  381,  381,  447,  383,   49,  450,
- /*  1200 */   381,   48,  215,   13,  443,  444,  445,    0,  447,  448,
- /*  1210 */   369,  381,   44,  381,  465,  466,  391,  101,   97,  470,
- /*  1220 */   471,  100,  381,  104,  383,   35,  107,  474,  414,   22,
- /*  1230 */   160,   35,  418,   44,  333,  421,  422,  423,  424,  425,
- /*  1240 */   426,   35,  428,  463,  343,  104,  272,  433,  107,  435,
- /*  1250 */     0,    0,  346,  439,  440,  414,   44,  136,  137,  418,
- /*  1260 */    44,   13,  421,  422,  423,  424,  425,  426,  104,  428,
- /*  1270 */   369,  107,   22,   22,  433,   44,  435,   44,  274,  457,
- /*  1280 */   439,  440,  381,   35,  383,   47,  369,  166,  167,  391,
- /*  1290 */   101,   35,  346,  172,  173,  341,  129,  130,  131,  132,
- /*  1300 */   133,  134,  135,    1,    2,   44,  343,  186,   44,  188,
- /*  1310 */   380,   44,  391,  101,   44,  414,   44,  101,   44,  418,
- /*  1320 */   449,  168,  421,  422,  423,  424,  425,  426,   72,  428,
- /*  1330 */    44,    0,  101,   44,  101,   44,  215,  216,  100,  218,
+ /*  1130 */   233,  234,  235,   12,   13,  381,  209,   97,    0,  333,
+ /*  1140 */   333,   20,  239,   22,  472,  473,  333,  333,  447,   44,
+ /*  1150 */   381,  450,  249,  333,   33,  381,   35,  343,  381,  345,
+ /*  1160 */   166,   44,  381,  333,  333,  333,  465,  466,  111,   13,
+ /*  1170 */   362,  470,  471,  365,  333,  104,  104,   35,  107,  107,
+ /*  1180 */    59,   13,   44,  369,  343,    0,  345,  381,  381,   44,
+ /*  1190 */   405,   35,  253,   72,  381,  381,  104,  383,  166,  107,
+ /*  1200 */   200,  381,  202,   35,  443,  444,  445,   22,  447,  448,
+ /*  1210 */   369,  381,  381,  381,   44,   44,   44,  160,   97,    0,
+ /*  1220 */    44,  100,  381,    0,  383,  136,  137,  370,  414,  358,
+ /*  1230 */     1,    2,  418,   44,  333,  421,  422,  423,  424,  425,
+ /*  1240 */   426,   22,  428,   35,  343,   22,  101,  433,   44,  435,
+ /*  1250 */    47,   44,   65,  439,  440,  414,   44,  136,  137,  418,
+ /*  1260 */    44,   44,  421,  422,  423,  424,  425,  426,  391,  428,
+ /*  1270 */   369,  101,  101,  101,  433,   44,  435,  101,   44,  334,
+ /*  1280 */   439,  440,  381,  346,  383,  474,  463,  166,  167,  369,
+ /*  1290 */   101,  457,  346,  172,  173,  391,  129,  130,  131,  132,
+ /*  1300 */   133,  134,  135,  100,  341,  101,  380,  186,  101,  188,
+ /*  1310 */   391,   44,   13,  101,   44,  414,   44,  101,  101,  418,
+ /*  1320 */     0,   44,  421,  422,  423,  424,  425,  426,  343,  428,
+ /*  1330 */   188,  449,  101,   44,   35,  101,  215,  216,   44,  218,
  /*  1340 */   219,  220,  221,  222,  223,  224,  225,  226,  227,  228,
- /*  1350 */   229,  230,  231,  232,  233,  234,  235,  441,  467,   12,
- /*  1360 */    13,   44,  101,  462,   44,  101,   44,  333,  101,   22,
- /*  1370 */    44,  101,   44,  101,   13,  101,  451,  343,  254,  345,
- /*  1380 */    33,   50,   35,  416,  188,   49,  415,  101,  184,  388,
- /*  1390 */   101,  402,  101,  276,  188,   42,   35,  333,   20,  391,
- /*  1400 */   388,  165,  386,  369,   20,  342,   59,  343,  342,  345,
- /*  1410 */   388,  386,  386,   98,  354,  381,  342,  383,  101,   72,
- /*  1420 */    96,  101,  353,  101,   95,  352,  342,  101,  342,  101,
- /*  1430 */   342,   20,  335,  369,   48,  339,  335,  339,   20,  409,
- /*  1440 */   350,   20,  383,  350,   97,  381,   20,  383,  414,  344,
- /*  1450 */    20,  401,  418,  350,  344,  421,  422,  423,  424,  425,
- /*  1460 */   426,  350,  428,  350,  350,  333,   53,  433,  342,  435,
- /*  1470 */   350,  335,  347,  439,  440,  343,  347,  369,  414,  335,
- /*  1480 */   381,  369,  418,  381,  369,  421,  422,  423,  424,  425,
- /*  1490 */   426,  369,  428,  342,  413,  203,  369,  433,  100,  435,
- /*  1500 */   369,  369,  369,  439,  440,  411,  369,  369,  369,  381,
- /*  1510 */   348,  369,  408,  381,  191,  383,  409,  192,  383,  348,
- /*  1520 */   190,  342,  269,  261,  456,  407,  260,  381,  456,  406,
- /*  1530 */   396,  381,  177,  186,  391,  188,  391,  381,  381,  459,
- /*  1540 */   458,  456,  271,  270,  396,  454,  414,  455,  255,  273,
- /*  1550 */   418,  475,  275,  421,  422,  423,  424,  425,  426,  453,
- /*  1560 */   428,  278,  215,  216,  469,  433,  343,  435,  416,  251,
- /*  1570 */    20,  439,  440,  333,  420,  228,  229,  230,  231,  232,
- /*  1580 */   233,  234,  348,  343,  342,  344,   20,  348,  394,  396,
- /*  1590 */   381,  381,  381,  333,  381,  381,  396,  170,  468,  381,
- /*  1600 */   348,  393,  365,  343,  100,  348,  343,  438,  100,  369,
- /*  1610 */   381,  342,   36,  359,    0,  336,  373,  335,  403,  397,
- /*  1620 */     0,  381,  397,  383,    0,  331,  348,  410,  363,  369,
- /*  1630 */   349,  363,   42,    0,   35,   35,  208,   35,  363,   35,
- /*  1640 */   208,  381,   35,  383,    0,   35,  208,    0,  208,    0,
- /*  1650 */    35,    0,   22,    0,  414,  195,   35,  188,  418,  186,
- /*  1660 */     0,  421,  422,  423,  424,  425,  426,    0,  428,    0,
- /*  1670 */   182,  181,    0,  433,  414,  435,   47,  333,  418,  439,
+ /*  1350 */   229,  230,  231,  232,  233,  234,  235,  467,  441,   12,
+ /*  1360 */    13,   13,  451,  462,  254,  416,   49,  333,  101,   22,
+ /*  1370 */    50,  101,  415,  101,  184,  402,   42,  343,  101,  345,
+ /*  1380 */    33,  276,   35,   35,  388,   20,  391,  388,  386,  165,
+ /*  1390 */   101,  274,   20,  342,  342,  101,  188,  333,  388,  386,
+ /*  1400 */   386,   98,  215,  369,  354,   96,   59,  343,  353,  345,
+ /*  1410 */   342,   95,  352,   20,  335,  381,  342,  383,  342,   72,
+ /*  1420 */   342,   48,  339,  335,  339,   20,  409,  350,   20,  383,
+ /*  1430 */   350,   20,   20,  369,  344,  344,  401,  350,   53,  342,
+ /*  1440 */   342,  347,  347,  350,   97,  381,  350,  383,  414,  350,
+ /*  1450 */   350,  335,  418,  369,  335,  421,  422,  423,  424,  425,
+ /*  1460 */   426,  203,  428,  381,  100,  333,  413,  433,  409,  435,
+ /*  1470 */   369,  369,  369,  439,  440,  343,  369,  369,  414,  348,
+ /*  1480 */   369,  369,  418,  369,  369,  421,  422,  423,  424,  425,
+ /*  1490 */   426,  369,  428,  192,  381,  191,  348,  433,  190,  435,
+ /*  1500 */   342,  369,  261,  439,  440,  381,  383,  381,  260,  381,
+ /*  1510 */   456,  391,  408,  381,  456,  383,  407,  411,  391,  269,
+ /*  1520 */   381,  396,  381,  177,  406,  396,  271,  270,  459,  255,
+ /*  1530 */   455,  458,  278,  186,  275,  188,  475,  456,  273,  251,
+ /*  1540 */   454,  343,   20,  342,  344,  348,  414,  348,   20,  394,
+ /*  1550 */   418,  416,  170,  421,  422,  423,  424,  425,  426,  396,
+ /*  1560 */   428,  381,  215,  216,  420,  433,  393,  435,  348,  381,
+ /*  1570 */   396,  439,  440,  333,  381,  228,  229,  230,  231,  232,
+ /*  1580 */   233,  234,  343,  343,  348,  100,  453,  381,  438,  100,
+ /*  1590 */   381,  381,  365,  333,  373,  381,  359,  348,  342,  469,
+ /*  1600 */    36,  335,  468,  343,  336,  403,  410,  363,  397,  369,
+ /*  1610 */   397,  349,  331,    0,  363,  363,    0,    0,   42,    0,
+ /*  1620 */    35,  381,  208,  383,   35,   35,   35,  208,    0,  369,
+ /*  1630 */    35,   35,  208,    0,  208,    0,   35,    0,    0,   35,
+ /*  1640 */    22,  381,  195,  383,  188,  186,    0,    0,    0,  182,
+ /*  1650 */   181,    0,    0,   47,  414,    0,    0,    0,  418,    0,
+ /*  1660 */     0,  421,  422,  423,  424,  425,  426,   42,  428,    0,
+ /*  1670 */     0,    0,    0,  433,  414,  435,  155,  333,  418,  439,
  /*  1680 */   440,  421,  422,  423,  424,  425,  426,  343,  428,    0,
- /*  1690 */     0,    0,    0,  433,   42,  435,    0,    0,    0,  439,
- /*  1700 */   440,  333,    0,    0,    0,    0,    0,  155,   35,    0,
- /*  1710 */   155,  343,    0,  369,    0,    0,    0,    0,    0,    0,
+ /*  1690 */     0,    0,  155,  433,    0,  435,   35,    0,    0,  439,
+ /*  1700 */   440,  333,    0,    0,    0,    0,    0,    0,    0,    0,
+ /*  1710 */     0,  343,    0,  369,    0,    0,    0,   42,    0,    0,
  /*  1720 */     0,    0,    0,    0,    0,  381,    0,  383,    0,    0,
- /*  1730 */     0,    0,    0,    0,    0,   42,    0,  369,    0,    0,
- /*  1740 */     0,    0,  139,    0,    0,    0,   59,   22,   48,  381,
- /*  1750 */     0,  383,    0,   59,   59,   48,    0,   22,  414,   22,
- /*  1760 */     0,   14,  418,  177,    0,  421,  422,  423,  424,  425,
- /*  1770 */   426,   44,  428,    0,    0,    0,    0,   35,  333,  435,
- /*  1780 */     0,    0,  414,  439,  440,   39,  418,   39,  343,  421,
- /*  1790 */   422,  423,  424,  425,  426,   42,  428,   39,   47,   40,
- /*  1800 */     0,  333,   47,  435,   47,    0,   35,  439,  440,   39,
- /*  1810 */    49,  343,    0,   35,  369,   66,   49,    0,   39,   35,
- /*  1820 */     0,   49,   39,   35,  333,   39,  381,    0,  383,    0,
- /*  1830 */     0,   49,    0,   35,  343,   22,    0,  369,   35,   35,
- /*  1840 */    44,   35,   35,   35,   35,   35,   44,  333,   22,  381,
- /*  1850 */    35,  383,    0,   35,   35,   22,    0,  343,    0,  414,
- /*  1860 */   369,  107,   22,  418,   22,   51,  421,  422,  423,  424,
- /*  1870 */   425,  426,  381,  428,  383,  109,   35,    0,   35,    0,
- /*  1880 */   435,   35,  414,  369,  439,  440,  418,    0,   22,  421,
- /*  1890 */   422,  423,  424,  425,  426,  381,  428,  383,   20,   35,
- /*  1900 */    35,  100,   35,  435,    0,  414,  193,   35,  440,  418,
- /*  1910 */   333,  100,  421,  422,  423,  424,  425,  426,   22,  428,
- /*  1920 */   343,    0,  101,   22,    0,    0,    3,   44,  414,  101,
- /*  1930 */   168,  256,  418,   48,  100,  421,  422,  423,  424,  425,
- /*  1940 */   426,   48,  428,  175,  100,  168,  369,   98,  101,   96,
- /*  1950 */    44,  374,   44,  168,  189,  464,  101,  100,  381,  170,
- /*  1960 */   383,  100,  100,   44,  101,   47,  256,    3,  100,   47,
- /*  1970 */    44,  101,  101,   44,   35,  333,   35,   35,   35,   35,
- /*  1980 */    35,  101,  101,    0,   47,  343,    0,  473,    0,    0,
- /*  1990 */    44,  414,  100,   39,   47,  418,  333,   47,  421,  422,
- /*  2000 */   423,  424,  425,  426,  101,  428,  343,  100,  171,  333,
- /*  2010 */   101,  369,  100,  100,  100,    0,   39,  169,   47,  343,
- /*  2020 */   110,    2,   44,  381,   22,  383,   98,  100,   98,  237,
- /*  2030 */    22,  215,  369,  100,   47,  101,  101,  374,  100,   47,
- /*  2040 */   250,  101,  217,  111,  381,  369,  383,  256,  100,  100,
- /*  2050 */   374,  100,   35,  101,  101,  100,  414,  381,   35,  383,
- /*  2060 */   418,    1,  100,  421,  422,  423,  424,  425,  426,  101,
- /*  2070 */   428,   35,  430,  100,   35,  101,  100,  414,   35,   19,
- /*  2080 */   101,  418,  100,   35,  421,  422,  423,  424,  425,  426,
- /*  2090 */   414,  428,  101,   33,  418,  100,  333,  421,  422,  423,
- /*  2100 */   424,  425,  426,  122,  428,  100,  343,  122,   44,   49,
- /*  2110 */   100,   35,  122,  100,   22,   55,   56,   57,   58,   59,
- /*  2120 */   333,  122,   66,   65,   35,   35,   35,   72,   35,   35,
- /*  2130 */   343,   35,  369,   35,   35,   35,   35,   35,   94,   44,
- /*  2140 */    35,   35,   22,   35,  381,   35,  383,   35,   35,   72,
- /*  2150 */    35,   35,   22,   35,   35,    0,  369,   35,   35,   99,
- /*  2160 */    49,    0,  102,   39,   35,   39,    0,   35,  381,    0,
- /*  2170 */   383,   35,    0,   39,   49,   35,   49,  414,   39,   35,
- /*  2180 */    49,  418,    0,  333,  421,  422,  423,  424,  425,  426,
- /*  2190 */    22,  428,   21,  343,  134,   22,   22,   21,   20,  476,
- /*  2200 */   476,  414,  476,  476,  476,  418,  476,  333,  421,  422,
- /*  2210 */   423,  424,  425,  426,  476,  428,  476,  343,  476,  369,
- /*  2220 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  169,
- /*  2230 */   476,  381,  476,  383,  174,  476,  476,  476,  476,  476,
- /*  2240 */   476,  476,  476,  369,  476,  476,  476,  476,  476,  476,
- /*  2250 */   476,  476,  476,  333,  194,  381,  476,  383,  476,  476,
- /*  2260 */   476,  476,  476,  343,  414,  476,  476,  476,  418,  333,
- /*  2270 */   476,  421,  422,  423,  424,  425,  426,  476,  428,  343,
- /*  2280 */   476,  476,  476,  476,  476,  476,  476,  476,  414,  369,
- /*  2290 */   476,  476,  418,  476,  476,  421,  422,  423,  424,  425,
- /*  2300 */   426,  381,  428,  383,  476,  369,  476,  476,  476,  476,
- /*  2310 */   476,  476,  476,  476,  476,  476,  476,  381,  476,  383,
- /*  2320 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  333,
- /*  2330 */   476,  476,  476,  476,  414,  476,  476,  476,  418,  343,
- /*  2340 */   476,  421,  422,  423,  424,  425,  426,  476,  428,  476,
- /*  2350 */   414,  476,  476,  333,  418,  476,  476,  421,  422,  423,
- /*  2360 */   424,  425,  426,  343,  428,  369,  476,  476,  476,  476,
- /*  2370 */   476,  476,  476,  476,  476,  476,  476,  381,  476,  383,
- /*  2380 */   333,  476,  476,  476,  476,  476,  476,  476,  476,  369,
- /*  2390 */   343,  476,  476,  476,  476,  476,  476,  476,  476,  333,
- /*  2400 */   476,  381,  476,  383,  476,  476,  476,  476,  476,  343,
- /*  2410 */   414,  476,  476,  476,  418,  476,  369,  421,  422,  423,
- /*  2420 */   424,  425,  426,  476,  428,  476,  476,  476,  381,  476,
- /*  2430 */   383,  476,  476,  476,  414,  369,  476,  476,  418,  476,
- /*  2440 */   476,  421,  422,  423,  424,  425,  426,  381,  428,  383,
- /*  2450 */   333,  476,  476,  476,  476,  476,  476,  476,  476,  476,
- /*  2460 */   343,  414,  476,  476,  476,  418,  333,  476,  421,  422,
- /*  2470 */   423,  424,  425,  426,  476,  428,  343,  476,  476,  476,
- /*  2480 */   414,  476,  476,  476,  418,  476,  369,  421,  422,  423,
- /*  2490 */   424,  425,  426,  476,  428,  476,  333,  476,  381,  476,
- /*  2500 */   383,  476,  369,  476,  476,  476,  343,  476,  476,  476,
- /*  2510 */   476,  476,  476,  476,  381,  476,  383,  333,  476,  476,
- /*  2520 */   476,  476,  476,  476,  476,  476,  476,  343,  476,  476,
- /*  2530 */   476,  414,  369,  476,  476,  418,  476,  476,  421,  422,
- /*  2540 */   423,  424,  425,  426,  381,  428,  383,  414,  476,  476,
- /*  2550 */   476,  418,  476,  369,  421,  422,  423,  424,  425,  426,
- /*  2560 */   476,  428,  476,  333,  476,  381,  476,  383,  476,  476,
- /*  2570 */   476,  476,  476,  343,  476,  476,  476,  414,  476,  476,
- /*  2580 */   476,  418,  476,  476,  421,  422,  423,  424,  425,  426,
- /*  2590 */   476,  428,  476,  476,  476,  476,  476,  476,  414,  369,
- /*  2600 */   476,  476,  418,  476,  476,  421,  422,  423,  424,  425,
- /*  2610 */   426,  381,  428,  383,  476,  476,  476,  476,  476,  476,
- /*  2620 */   476,  476,  476,  333,  476,  476,  476,  476,  476,  476,
- /*  2630 */   476,  476,  476,  343,  476,  476,  333,  476,  476,  476,
- /*  2640 */   476,  476,  476,  476,  414,  476,  343,  476,  418,  476,
- /*  2650 */   476,  421,  422,  423,  424,  425,  426,  476,  428,  369,
- /*  2660 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  476,
- /*  2670 */   476,  381,  369,  383,  476,  476,  476,  476,  476,  476,
- /*  2680 */   476,  476,  476,  476,  381,  476,  383,  333,  476,  476,
- /*  2690 */   476,  476,  476,  476,  476,  476,  476,  343,  476,  476,
- /*  2700 */   476,  476,  476,  476,  414,  476,  476,  476,  418,  476,
- /*  2710 */   476,  421,  422,  423,  424,  425,  426,  414,  428,  476,
- /*  2720 */   476,  418,  476,  369,  421,  422,  423,  424,  425,  426,
- /*  2730 */   476,  428,  476,  333,  476,  381,  476,  383,  476,  476,
- /*  2740 */   476,  476,  476,  343,  476,  476,  476,  476,  333,  476,
- /*  2750 */   476,  476,  476,  476,  476,  476,  476,  476,  343,  476,
- /*  2760 */   476,  476,  476,  476,  476,  476,  476,  476,  414,  369,
- /*  2770 */   476,  476,  418,  476,  476,  421,  422,  423,  424,  425,
- /*  2780 */   426,  381,  428,  383,  369,  476,  476,  476,  476,  476,
- /*  2790 */   476,  476,  476,  476,  333,  476,  381,  476,  383,  476,
- /*  2800 */   476,  476,  476,  476,  343,  476,  476,  476,  476,  333,
- /*  2810 */   476,  476,  476,  476,  414,  476,  476,  476,  418,  343,
- /*  2820 */   476,  421,  422,  423,  424,  425,  426,  476,  428,  414,
- /*  2830 */   369,  476,  476,  418,  476,  476,  421,  422,  423,  424,
- /*  2840 */   425,  426,  381,  428,  383,  369,  476,  476,  476,  476,
- /*  2850 */   476,  476,  476,  476,  476,  476,  476,  381,  476,  383,
- /*  2860 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  476,
- /*  2870 */   476,  476,  476,  476,  476,  414,  476,  476,  476,  418,
- /*  2880 */   476,  476,  421,  422,  423,  424,  425,  426,  476,  428,
- /*  2890 */   414,  476,  476,  476,  418,  476,  476,  421,  422,  423,
- /*  2900 */   424,  425,  426,  476,  428,
+ /*  1730 */     0,  139,   48,   22,   48,   22,    0,  369,   59,   22,
+ /*  1740 */     0,   59,    0,    0,   59,    0,   47,   47,   14,  381,
+ /*  1750 */   177,  383,    0,    0,    0,    0,    0,    0,  414,    0,
+ /*  1760 */    35,   66,  418,    0,    0,  421,  422,  423,  424,  425,
+ /*  1770 */   426,   39,  428,   42,   47,   40,   39,   44,  333,  435,
+ /*  1780 */    39,   35,  414,  439,  440,    0,  418,   49,  343,  421,
+ /*  1790 */   422,  423,  424,  425,  426,   35,  428,   39,   49,    0,
+ /*  1800 */    39,  333,   35,  435,   39,    0,   35,  439,  440,    0,
+ /*  1810 */     0,  343,   39,    0,  369,   49,    0,   49,   35,   22,
+ /*  1820 */     0,   35,   35,   35,  333,   35,  381,   35,  383,   35,
+ /*  1830 */    35,   35,    0,   44,  343,   44,   35,  369,   35,   22,
+ /*  1840 */    22,    0,    0,   22,   51,   22,   35,  333,  109,  381,
+ /*  1850 */     0,  383,    0,    0,  107,   22,   20,  343,    0,  414,
+ /*  1860 */   369,   35,   35,  418,   35,  374,  421,  422,  423,  424,
+ /*  1870 */   425,  426,  381,  428,  383,   35,  101,  100,   35,  100,
+ /*  1880 */   435,  193,  414,  369,  439,  440,  418,   22,  374,  421,
+ /*  1890 */   422,  423,  424,  425,  426,  381,  428,  383,   35,    0,
+ /*  1900 */    22,    0,    0,  435,   44,  414,    3,  100,  440,  418,
+ /*  1910 */   333,   96,  421,  422,  423,  424,  425,  426,  170,  428,
+ /*  1920 */   343,  256,   48,  101,  100,   48,  175,   47,  414,   98,
+ /*  1930 */   168,  101,  418,  189,   44,  421,  422,  423,  424,  425,
+ /*  1940 */   426,   44,  428,   47,  168,  101,  369,  100,  168,  100,
+ /*  1950 */    44,  100,   44,    3,   44,  101,  100,  256,  381,   35,
+ /*  1960 */   383,   35,  101,  101,   35,   35,   35,   35,   47,  101,
+ /*  1970 */   101,   44,   47,    0,    0,  333,    0,    0,   39,  100,
+ /*  1980 */    47,  100,    0,  101,  100,  343,  101,  100,  100,  100,
+ /*  1990 */   171,  414,   39,   47,  169,  418,  333,    2,  421,  422,
+ /*  2000 */   423,  424,  425,  426,  110,  428,  343,   44,   22,  333,
+ /*  2010 */    98,  369,   98,  215,   47,  237,  100,   47,  101,  343,
+ /*  2020 */   100,   22,  100,  381,  250,  383,  111,   35,  256,  101,
+ /*  2030 */    22,  217,  369,   35,   44,  101,  100,  374,  100,   35,
+ /*  2040 */   100,  464,  101,  101,  381,  369,  383,  100,   35,  101,
+ /*  2050 */   101,  100,  100,   35,   35,  101,  414,  381,  100,  383,
+ /*  2060 */   418,  101,  100,  421,  422,  423,  424,  425,  426,  100,
+ /*  2070 */   428,  100,   35,   35,   65,  100,   66,  414,  122,  122,
+ /*  2080 */    35,  418,   35,  333,  421,  422,  423,  424,  425,  426,
+ /*  2090 */   414,  428,   35,  343,  418,   35,   35,  421,  422,  423,
+ /*  2100 */   424,  425,  426,  333,  428,  122,  430,  122,   35,   35,
+ /*  2110 */    35,   35,   72,  343,   44,  473,   94,   35,   35,  369,
+ /*  2120 */    35,   22,   35,   35,  374,   35,   72,   35,   35,   35,
+ /*  2130 */    35,  381,   35,  383,   22,   35,    0,   35,   49,  369,
+ /*  2140 */    39,    0,   35,   39,  374,   49,    0,   49,   35,   39,
+ /*  2150 */     0,  381,   35,  383,   49,   39,    0,   35,   35,    0,
+ /*  2160 */    22,   21,  476,   21,  414,   22,   22,   20,  418,  476,
+ /*  2170 */   476,  421,  422,  423,  424,  425,  426,  476,  428,  476,
+ /*  2180 */   476,  476,  476,  476,  414,  476,    1,  476,  418,  476,
+ /*  2190 */   476,  421,  422,  423,  424,  425,  426,  476,  428,  476,
+ /*  2200 */   333,  476,  476,  476,   19,  476,  476,  476,  476,  476,
+ /*  2210 */   343,  476,  476,  476,  476,  476,  333,  476,   33,  476,
+ /*  2220 */   476,  476,  476,  476,  476,  476,  343,  476,  476,  476,
+ /*  2230 */   476,  476,  476,  476,   49,  476,  369,  476,  476,  476,
+ /*  2240 */    55,   56,   57,   58,   59,  476,  333,  476,  381,  476,
+ /*  2250 */   383,  476,  369,  476,  476,  476,  343,  476,  476,  476,
+ /*  2260 */   476,  476,  476,  476,  381,  476,  383,  476,  476,  476,
+ /*  2270 */   476,  476,  476,  476,  476,  476,  333,  476,  476,  476,
+ /*  2280 */   476,  414,  369,  476,   99,  418,  343,  102,  421,  422,
+ /*  2290 */   423,  424,  425,  426,  381,  428,  383,  414,  476,  476,
+ /*  2300 */   476,  418,  476,  476,  421,  422,  423,  424,  425,  426,
+ /*  2310 */   476,  428,  369,  476,  476,  476,  476,  476,  476,  134,
+ /*  2320 */   476,  476,  476,  476,  381,  476,  383,  414,  476,  476,
+ /*  2330 */   476,  418,  476,  476,  421,  422,  423,  424,  425,  426,
+ /*  2340 */   333,  428,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2350 */   343,  476,  476,  333,  169,  476,  476,  414,  476,  174,
+ /*  2360 */   476,  418,  476,  343,  421,  422,  423,  424,  425,  426,
+ /*  2370 */   476,  428,  476,  476,  476,  476,  369,  476,  476,  194,
+ /*  2380 */   476,  476,  476,  476,  476,  476,  476,  476,  381,  369,
+ /*  2390 */   383,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2400 */   476,  381,  476,  383,  333,  476,  476,  476,  476,  476,
+ /*  2410 */   476,  476,  476,  476,  343,  476,  476,  333,  476,  476,
+ /*  2420 */   476,  414,  476,  476,  476,  418,  476,  343,  421,  422,
+ /*  2430 */   423,  424,  425,  426,  414,  428,  476,  476,  418,  476,
+ /*  2440 */   369,  421,  422,  423,  424,  425,  426,  476,  428,  476,
+ /*  2450 */   476,  476,  381,  369,  383,  476,  476,  476,  476,  476,
+ /*  2460 */   476,  476,  476,  476,  476,  381,  476,  383,  476,  476,
+ /*  2470 */   476,  476,  476,  476,  476,  476,  476,  476,  333,  476,
+ /*  2480 */   476,  476,  476,  476,  476,  414,  476,  476,  343,  418,
+ /*  2490 */   476,  476,  421,  422,  423,  424,  425,  426,  414,  428,
+ /*  2500 */   476,  476,  418,  476,  476,  421,  422,  423,  424,  425,
+ /*  2510 */   426,  476,  428,  476,  369,  476,  476,  476,  476,  476,
+ /*  2520 */   476,  476,  476,  476,  476,  476,  381,  476,  383,  333,
+ /*  2530 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  343,
+ /*  2540 */   476,  476,  476,  476,  333,  476,  476,  476,  476,  476,
+ /*  2550 */   476,  476,  476,  476,  343,  476,  476,  476,  476,  414,
+ /*  2560 */   476,  476,  476,  418,  476,  369,  421,  422,  423,  424,
+ /*  2570 */   425,  426,  476,  428,  476,  476,  476,  381,  476,  383,
+ /*  2580 */   369,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2590 */   333,  476,  381,  476,  383,  476,  476,  476,  476,  476,
+ /*  2600 */   343,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2610 */   414,  476,  476,  476,  418,  476,  476,  421,  422,  423,
+ /*  2620 */   424,  425,  426,  476,  428,  414,  369,  476,  476,  418,
+ /*  2630 */   476,  476,  421,  422,  423,  424,  425,  426,  381,  428,
+ /*  2640 */   383,  333,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2650 */   476,  343,  476,  476,  476,  476,  476,  476,  333,  476,
+ /*  2660 */   476,  476,  476,  476,  476,  476,  476,  476,  343,  476,
+ /*  2670 */   476,  414,  476,  476,  476,  418,  476,  369,  421,  422,
+ /*  2680 */   423,  424,  425,  426,  476,  428,  476,  333,  476,  381,
+ /*  2690 */   476,  383,  476,  476,  369,  476,  476,  343,  476,  476,
+ /*  2700 */   476,  476,  476,  476,  476,  476,  381,  476,  383,  476,
+ /*  2710 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2720 */   476,  476,  414,  369,  476,  476,  418,  476,  476,  421,
+ /*  2730 */   422,  423,  424,  425,  426,  381,  428,  383,  476,  414,
+ /*  2740 */   476,  476,  476,  418,  476,  476,  421,  422,  423,  424,
+ /*  2750 */   425,  426,  476,  428,  476,  333,  476,  476,  476,  476,
+ /*  2760 */   476,  476,  476,  476,  476,  343,  476,  476,  414,  476,
+ /*  2770 */   476,  476,  418,  476,  333,  421,  422,  423,  424,  425,
+ /*  2780 */   426,  476,  428,  476,  343,  476,  476,  333,  476,  476,
+ /*  2790 */   476,  369,  476,  476,  476,  476,  476,  343,  476,  476,
+ /*  2800 */   476,  476,  476,  381,  476,  383,  476,  476,  476,  476,
+ /*  2810 */   369,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2820 */   476,  476,  381,  369,  383,  476,  476,  476,  476,  476,
+ /*  2830 */   476,  476,  476,  476,  476,  381,  414,  383,  476,  476,
+ /*  2840 */   418,  476,  476,  421,  422,  423,  424,  425,  426,  476,
+ /*  2850 */   428,  476,  476,  476,  476,  414,  476,  476,  476,  418,
+ /*  2860 */   333,  476,  421,  422,  423,  424,  425,  426,  414,  428,
+ /*  2870 */   343,  476,  418,  333,  476,  421,  422,  423,  424,  425,
+ /*  2880 */   426,  476,  428,  343,  476,  476,  476,  476,  333,  476,
+ /*  2890 */   476,  476,  476,  476,  476,  476,  369,  476,  343,  476,
+ /*  2900 */   476,  476,  476,  476,  476,  476,  476,  476,  381,  369,
+ /*  2910 */   383,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  2920 */   476,  381,  476,  383,  369,  476,  476,  476,  476,  476,
+ /*  2930 */   476,  476,  476,  476,  333,  476,  381,  476,  383,  476,
+ /*  2940 */   476,  414,  476,  476,  343,  418,  476,  476,  421,  422,
+ /*  2950 */   423,  424,  425,  426,  414,  428,  476,  476,  418,  476,
+ /*  2960 */   476,  421,  422,  423,  424,  425,  426,  476,  428,  414,
+ /*  2970 */   369,  476,  476,  418,  476,  476,  421,  422,  423,  424,
+ /*  2980 */   425,  426,  381,  428,  383,  476,  476,  476,  476,  476,
+ /*  2990 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  3000 */   476,  476,  476,  476,  476,  476,  476,  476,  476,  476,
+ /*  3010 */   476,  476,  476,  476,  476,  414,  476,  476,  476,  418,
+ /*  3020 */   476,  476,  421,  422,  423,  424,  425,  426,  476,  428,
 };
 #define YY_SHIFT_COUNT    (761)
 #define YY_SHIFT_MIN      (0)
-#define YY_SHIFT_MAX      (2182)
+#define YY_SHIFT_MAX      (2185)
 static const unsigned short int yy_shift_ofst[] = {
  /*     0 */   550,    0,  224,    0,  449,  449,  449,  449,  449,  449,
  /*    10 */   449,  449,  449,  449,  449,  449,  673,  897,  897, 1121,
  /*    20 */   897,  897,  897,  897,  897,  897,  897,  897,  897,  897,
  /*    30 */   897,  897,  897,  897,  897,  897,  897,  897,  897,  897,
- /*    40 */   897,  897,  897,  897,  897,  897,  897,  897,  897,  457,
- /*    50 */   492,  142,  145,  147,  107,  173,  107,  145,  145, 1347,
- /*    60 */  1347, 1347,  107, 1347, 1347,  501,  107,   18,  447,   47,
- /*    70 */    47,  447,  235,  235,    3,  286,    7,    7,   47,   47,
- /*    80 */    47,   47,   47,   47,   47,   51,   47,   47,  132,   18,
- /*    90 */    47,   47,  228,   47,   18,   47,   51,   47,   51,   18,
- /*   100 */    47,   47,   18,   47,   18,   18,   18,   47,  339,  223,
+ /*    40 */   897,  897,  897,  897,  897,  897,  897,  897,  897,  897,
+ /*    50 */   897,  452,  457,  142,  145,  147,   30,  107,   30,  145,
+ /*    60 */   145, 1347,   30, 1347, 1347,  501,   30,   56,  447,  344,
+ /*    70 */   344,  447,  235,  235,   18,  286,  150,  150,  344,  344,
+ /*    80 */   344,  344,  344,  344,  344,  369,  344,  344,  412,   56,
+ /*    90 */   344,  344,  484,  344,   56,  344,  369,  344,  369,   56,
+ /*   100 */   344,  344,   56,  344,   56,   56,   56,  344,  538,  223,
  /*   110 */   189,  189,  333,  114,  443,  443,  443,  443,  443,  443,
  /*   120 */   443,  443,  443,  443,  443,  443,  443,  443,  443,  443,
- /*   130 */   443,  443,  443,  843,  325,    3,  286,  392,  220,  220,
- /*   140 */   220,  417,  369,  369,  392,  344,  344,  344,  132,  365,
- /*   150 */   297,   18,  439,   18,  439,  439,  503,  558,   35,   35,
- /*   160 */    35,   35,   35,   35,   35,   35, 2060,  217,  288,  483,
- /*   170 */   522,  260,  313,  243,  384,  940,  576,  484,  890,  751,
- /*   180 */   528,  810,  824,   60,  974,  824, 1065,  791,  275, 1124,
- /*   190 */  1336, 1204, 1353, 1378, 1353, 1236, 1384, 1384, 1353, 1236,
- /*   200 */  1236, 1315, 1324, 1384, 1329, 1384, 1384, 1384, 1411, 1386,
- /*   210 */  1411, 1386, 1418,  132, 1421,  132, 1426, 1430,  132, 1426,
- /*   220 */   132,  132,  132, 1384,  132, 1413, 1413, 1411,   18,   18,
- /*   230 */    18,   18,   18,   18,   18,   18,   18,   18,   18, 1384,
- /*   240 */  1411,  439,  439,  439, 1292, 1398, 1418,  339, 1325, 1323,
- /*   250 */  1421,  339, 1330, 1384, 1378, 1378,  439, 1262, 1266,  439,
- /*   260 */  1262, 1266,  439,  439,   18, 1253, 1355, 1262, 1271, 1273,
- /*   270 */  1293, 1124, 1283, 1277, 1276, 1318,  344, 1550, 1384, 1426,
- /*   280 */   339,  339, 1566, 1266,  439,  439,  439,  439,  439, 1266,
- /*   290 */   439, 1427,  339,  503,  339,  344, 1504, 1508,  439,  558,
- /*   300 */  1384,  339, 1576, 1411, 2905, 2905, 2905, 2905, 2905, 2905,
- /*   310 */  2905, 2905, 2905,   34,  430,   15,  590,  734,   16,  839,
+ /*   130 */   443,  443,  443,  843,  325,   18,  286,  392,    1,    1,
+ /*   140 */     1,  539,  273,  273,  392,  627,  627,  627,  412,  615,
+ /*   150 */   529,   56,  699,   56,  699,  699,  715,  778,   35,   35,
+ /*   160 */    35,   35,   35,   35,   35,   35, 2185,  217,  288,  483,
+ /*   170 */   522,  260,  313,  243,  536, 1032,  490,  646,  868,  616,
+ /*   180 */   602,  916,  829,   60,  422,  829,  966,  939,  994, 1110,
+ /*   190 */  1317, 1190, 1334, 1365, 1334, 1224, 1372, 1372, 1334, 1224,
+ /*   200 */  1224, 1303, 1309, 1372, 1316, 1372, 1372, 1372, 1393, 1373,
+ /*   210 */  1393, 1373, 1405,  412, 1408,  412, 1411, 1412,  412, 1411,
+ /*   220 */   412,  412,  412, 1372,  412, 1385, 1385, 1393,   56,   56,
+ /*   230 */    56,   56,   56,   56,   56,   56,   56,   56,   56, 1372,
+ /*   240 */  1393,  699,  699,  699, 1258, 1364, 1405,  538, 1301, 1304,
+ /*   250 */  1408,  538, 1308, 1372, 1365, 1365,  699, 1241, 1248,  699,
+ /*   260 */  1241, 1248,  699,  699,   56, 1250, 1346, 1241, 1255, 1257,
+ /*   270 */  1274, 1110, 1254, 1259, 1265, 1288,  627, 1522, 1372, 1411,
+ /*   280 */   538,  538, 1528, 1248,  699,  699,  699,  699,  699, 1248,
+ /*   290 */   699, 1382,  538,  715,  538,  627, 1485, 1489,  699,  778,
+ /*   300 */  1372,  538, 1564, 1393, 3030, 3030, 3030, 3030, 3030, 3030,
+ /*   310 */  3030, 3030, 3030,   34,  430,   15,  590,  734,   16,  839,
  /*   320 */    79,  709,  792,  566,  856, 1023, 1023, 1023, 1023, 1023,
  /*   330 */  1023, 1023, 1023, 1023, 1167,  743,  253,  253,  402,   57,
- /*   340 */   758,  774,  949, 1067,  800,  980,   64,   64,  959, 1007,
- /*   350 */   387,  959,  959,  959, 1149,  641, 1116, 1140, 1147, 1070,
- /*   360 */  1168, 1078, 1119, 1141, 1164,  532, 1190, 1207, 1250, 1251,
- /*   370 */   990, 1189, 1212,  868, 1216, 1231, 1233,  943, 1004, 1117,
- /*   380 */  1153, 1261, 1264, 1267, 1270, 1272, 1274, 1302, 1289, 1196,
- /*   390 */  1206,  987, 1291, 1238, 1286, 1317, 1320, 1322, 1326, 1328,
- /*   400 */   190, 1248, 1361, 1256, 1331, 1614, 1620, 1624, 1590, 1633,
- /*   410 */  1599, 1428, 1600, 1602, 1604, 1432, 1644, 1607, 1610, 1438,
- /*   420 */  1647, 1440, 1649, 1615, 1651, 1630, 1653, 1621, 1460, 1469,
- /*   430 */  1473, 1660, 1667, 1669, 1488, 1490, 1672, 1689, 1629, 1690,
- /*   440 */  1691, 1692, 1652, 1696, 1697, 1698, 1702, 1703, 1704, 1705,
- /*   450 */  1706, 1552, 1673, 1709, 1555, 1712, 1714, 1715, 1716, 1717,
- /*   460 */  1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1728, 1729,
- /*   470 */  1730, 1693, 1731, 1732, 1733, 1734, 1736, 1738, 1725, 1739,
- /*   480 */  1740, 1741, 1603, 1743, 1744, 1735, 1700, 1737, 1707, 1745,
- /*   490 */  1687, 1742, 1750, 1694, 1752, 1695, 1756, 1760, 1753, 1746,
- /*   500 */  1727, 1751, 1755, 1747, 1757, 1773, 1759, 1748, 1774, 1775,
- /*   510 */  1776, 1758, 1586, 1764, 1780, 1781, 1749, 1800, 1805, 1771,
- /*   520 */  1761, 1770, 1812, 1778, 1767, 1779, 1817, 1784, 1772, 1783,
- /*   530 */  1820, 1788, 1782, 1786, 1827, 1829, 1830, 1832, 1766, 1754,
- /*   540 */  1798, 1813, 1836, 1803, 1804, 1806, 1807, 1808, 1809, 1810,
- /*   550 */  1796, 1802, 1815, 1818, 1826, 1819, 1852, 1833, 1856, 1840,
- /*   560 */  1814, 1858, 1842, 1841, 1877, 1843, 1879, 1846, 1887, 1866,
- /*   570 */  1878, 1864, 1865, 1867, 1821, 1801, 1904, 1762, 1811, 1713,
- /*   580 */  1872, 1896, 1921, 1765, 1901, 1777, 1789, 1924, 1925, 1785,
- /*   590 */  1768, 1923, 1883, 1675, 1834, 1828, 1844, 1885, 1849, 1893,
- /*   600 */  1853, 1847, 1906, 1908, 1855, 1857, 1861, 1862, 1863, 1919,
- /*   610 */  1918, 1922, 1868, 1926, 1710, 1870, 1871, 1964, 1929, 1791,
- /*   620 */  1939, 1941, 1942, 1943, 1944, 1945, 1880, 1881, 1937, 1790,
- /*   630 */  1946, 1947, 1983, 1986, 1988, 1989, 1892, 1954, 1751, 1950,
- /*   640 */  1907, 1903, 1909, 1912, 1913, 1837, 1914, 2015, 1977, 1848,
- /*   650 */  1927, 1910, 1751, 1971, 1978, 1928, 1792, 1930, 2019, 2002,
- /*   660 */  1816, 1933, 1934, 1938, 1935, 1948, 1940, 1987, 1949, 1951,
- /*   670 */  1992, 1952, 2008, 1825, 1955, 1932, 1953, 2017, 2023, 1962,
- /*   680 */  1968, 2036, 1973, 1974, 2039, 1976, 1979, 2043, 1982, 1991,
- /*   690 */  2048, 1995, 1981, 1985, 1990, 1999, 2005, 2064, 2010, 2076,
- /*   700 */  2013, 2064, 2064, 2092, 2056, 2058, 2089, 2090, 2091, 2093,
- /*   710 */  2094, 2096, 2098, 2099, 2100, 2101, 2055, 2044, 2095, 2102,
- /*   720 */  2105, 2106, 2120, 2108, 2110, 2112, 2077, 1796, 2113, 1802,
- /*   730 */  2115, 2116, 2118, 2119, 2130, 2122, 2155, 2123, 2111, 2124,
- /*   740 */  2161, 2129, 2125, 2126, 2166, 2132, 2127, 2134, 2169, 2136,
- /*   750 */  2131, 2139, 2172, 2140, 2144, 2182, 2168, 2171, 2173, 2174,
- /*   760 */  2176, 2178,
+ /*   340 */   441,  414,  838, 1040,  934,  636,   64,   64,  638,  606,
+ /*   350 */   903,  638,  638,  638,  417,  927, 1007, 1070, 1067, 1057,
+ /*   360 */  1138,  967, 1071, 1072, 1092, 1156, 1168, 1185, 1219, 1223,
+ /*   370 */  1000, 1145, 1170,  431, 1171, 1172, 1176, 1089, 1117, 1105,
+ /*   380 */   809, 1189, 1204, 1207, 1212, 1216, 1217, 1229, 1231, 1142,
+ /*   390 */  1208, 1187, 1234, 1203, 1267, 1270, 1272, 1277, 1289, 1294,
+ /*   400 */   190, 1299, 1348,  668, 1320, 1613, 1616, 1617, 1576, 1619,
+ /*   410 */  1585, 1414, 1589, 1590, 1591, 1419, 1628, 1595, 1596, 1424,
+ /*   420 */  1633, 1426, 1635, 1601, 1637, 1618, 1638, 1604, 1447, 1456,
+ /*   430 */  1459, 1646, 1647, 1648, 1467, 1469, 1651, 1652, 1606, 1655,
+ /*   440 */  1656, 1657, 1625, 1659, 1660, 1669, 1670, 1671, 1672, 1689,
+ /*   450 */  1690, 1521, 1661, 1691, 1537, 1694, 1697, 1698, 1702, 1703,
+ /*   460 */  1704, 1705, 1706, 1707, 1708, 1709, 1710, 1712, 1714, 1715,
+ /*   470 */  1716, 1675, 1718, 1719, 1720, 1721, 1722, 1723, 1711, 1724,
+ /*   480 */  1726, 1728, 1592, 1729, 1730, 1713, 1684, 1717, 1686, 1736,
+ /*   490 */  1679, 1725, 1740, 1682, 1742, 1685, 1743, 1745, 1731, 1732,
+ /*   500 */  1733, 1699, 1700, 1734, 1727, 1752, 1735, 1737, 1753, 1754,
+ /*   510 */  1755, 1741, 1573, 1756, 1757, 1759, 1695, 1763, 1764, 1746,
+ /*   520 */  1738, 1758, 1785, 1760, 1749, 1761, 1799, 1767, 1766, 1765,
+ /*   530 */  1805, 1771, 1768, 1773, 1809, 1810, 1813, 1816, 1739, 1747,
+ /*   540 */  1783, 1797, 1820, 1786, 1787, 1788, 1790, 1792, 1794, 1795,
+ /*   550 */  1789, 1791, 1796, 1801, 1817, 1803, 1832, 1818, 1841, 1821,
+ /*   560 */  1793, 1842, 1823, 1811, 1850, 1826, 1852, 1827, 1853, 1833,
+ /*   570 */  1836, 1829, 1840, 1843, 1775, 1777, 1858, 1762, 1779, 1688,
+ /*   580 */  1863, 1865, 1899, 1744, 1878, 1776, 1748, 1901, 1902, 1780,
+ /*   590 */  1751, 1903, 1860, 1665, 1807, 1822, 1824, 1874, 1831, 1877,
+ /*   600 */  1815, 1830, 1890, 1897, 1844, 1847, 1849, 1851, 1854, 1906,
+ /*   610 */  1880, 1896, 1856, 1908, 1701, 1861, 1862, 1950, 1910, 1772,
+ /*   620 */  1924, 1926, 1929, 1930, 1931, 1932, 1868, 1869, 1921, 1774,
+ /*   630 */  1927, 1925, 1973, 1974, 1976, 1977, 1879, 1939, 1699, 1933,
+ /*   640 */  1881, 1882, 1885, 1884, 1887, 1819, 1888, 1982, 1953, 1825,
+ /*   650 */  1889, 1894, 1699, 1946, 1963, 1912, 1778, 1914, 1995, 1986,
+ /*   660 */  1798, 1916, 1917, 1920, 1928, 1922, 1934, 1967, 1936, 1938,
+ /*   670 */  1970, 1941, 1999, 1814, 1940, 1915, 1942, 1992, 1998, 1947,
+ /*   680 */  1948, 2004, 1951, 1949, 2013, 1952, 1954, 2018, 1958, 1960,
+ /*   690 */  2019, 1962, 1956, 1957, 1983, 1985, 1969, 1990, 1971, 2037,
+ /*   700 */  1975, 1990, 1990, 2008, 2010, 2009, 2038, 2045, 2047, 2057,
+ /*   710 */  2060, 2061, 2073, 2074, 2075, 2076, 2040, 2022, 2070, 2082,
+ /*   720 */  2083, 2085, 2099, 2087, 2088, 2090, 2054, 1789, 2092, 1791,
+ /*   730 */  2093, 2094, 2095, 2097, 2112, 2100, 2136, 2102, 2089, 2101,
+ /*   740 */  2141, 2107, 2096, 2104, 2146, 2113, 2098, 2110, 2150, 2117,
+ /*   750 */  2105, 2116, 2156, 2122, 2123, 2159, 2138, 2140, 2143, 2144,
+ /*   760 */  2142, 2147,
 };
 #define YY_REDUCE_COUNT (312)
 #define YY_REDUCE_MIN   (-396)
-#define YY_REDUCE_MAX   (2476)
+#define YY_REDUCE_MAX   (2601)
 static const short yy_reduce_ofst[] = {
  /*     0 */  -140,  137,  -82,  358,  444,  578,  814,  841, 1034, 1064,
  /*    10 */  1132, 1240, 1260, 1344, 1368, 1445,  598, -333,  672, 1468,
- /*    20 */   901, 1491, 1514, 1577, 1642, 1663, 1676, 1763, 1787, 1850,
- /*    30 */  1874, 1920, 1936, 1996, 2020, 2047, 2066, 2117, 2133, 2163,
- /*    40 */  2184, 2230, 2290, 2303, 2354, 2400, 2415, 2461, 2476,  372,
- /*    50 */  -195, -396, -265,  487,  595,  607,  701,  481,  761, -362,
- /*    60 */  -355,  285,  749,  136,  612, -279,  -55,  256, -346, -167,
- /*    70 */    -5, -381, -334, -257, -205, -249,   45,  130,  300,  303,
- /*    80 */   356,  377,  409,  425,  432,  177,  469,  489,  252, -308,
- /*    90 */   521,  573,  170,  620,  172,  622,  261,  707,  396,  682,
- /*   100 */   739,  745,  278,  793,  715,  321,  781,  795, -266,    8,
- /*   110 */  -350, -350,  115, -238,  161,  214,  257,  315,  374,  433,
- /*   120 */   445,  463,  585,  587,  617,  677,  681,  806,  807,  813,
- /*   130 */   819,  830,  832,  -86, -289, -111,   23,  154, -289,   70,
- /*   140 */   188,  -25,  480,  511,  380,  416,  498,  704,  381, -354,
- /*   150 */   426,  513,  690,  502,  733,  798,  808,  353, -369, -365,
- /*   160 */   324,  370,  458,  505,  686,  458,  764,  368,  621,  825,
- /*   170 */   753,  780,  906,  822,  917,  917,  946,  898,  954,  963,
- /*   180 */   930,  921,  871,  871,  891,  871,  916,  925,  917,  967,
- /*   190 */   971,  989, 1001, 1008, 1012, 1016, 1063, 1066, 1022, 1025,
- /*   200 */  1026, 1060, 1069, 1074, 1073, 1084, 1086, 1088, 1097, 1096,
- /*   210 */  1101, 1098, 1030, 1090, 1059, 1093, 1105, 1050, 1103, 1110,
- /*   220 */  1111, 1113, 1114, 1126, 1120, 1125, 1129, 1136, 1108, 1112,
- /*   230 */  1115, 1122, 1127, 1131, 1133, 1137, 1138, 1139, 1142, 1151,
- /*   240 */  1144, 1099, 1102, 1128, 1081, 1094, 1107, 1162, 1104, 1118,
- /*   250 */  1135, 1171, 1123, 1179, 1143, 1145, 1146, 1068, 1134, 1150,
- /*   260 */  1072, 1148, 1156, 1157,  917, 1080, 1082, 1085, 1092, 1091,
- /*   270 */  1106, 1152, 1076, 1095, 1130,  871, 1223, 1154, 1242, 1241,
- /*   280 */  1234, 1239, 1194, 1193, 1209, 1210, 1211, 1213, 1214, 1200,
- /*   290 */  1218, 1208, 1252, 1237, 1257, 1263, 1169, 1243, 1229, 1254,
- /*   300 */  1269, 1278, 1279, 1282, 1215, 1217, 1222, 1225, 1265, 1268,
- /*   310 */  1275, 1281, 1294,
+ /*    20 */   901, 1491, 1514, 1577, 1642, 1663, 1676, 1750, 1770, 1867,
+ /*    30 */  1883, 1913, 1943, 2007, 2020, 2071, 2084, 2145, 2196, 2211,
+ /*    40 */  2257, 2308, 2325, 2354, 2422, 2441, 2454, 2527, 2540, 2555,
+ /*    50 */  2601,  372, -195, -396, -265,  487,  595,  607,  701,  481,
+ /*    60 */   761, -362,  191, -365,  136, -279,  -55,  256, -372, -167,
+ /*    70 */    -5, -381, -273, -257,   14, -321, -337,  203,   45,  285,
+ /*    80 */   300,  356,  377,  409,  425,  177,  432,  489,  446,  279,
+ /*    90 */   521,  573,  125,  620,  321,  622,  396,  687,  436,  257,
+ /*   100 */   707,  716,  460,  739,  434,  506,  679,  746, -266,    8,
+ /*   110 */  -350, -350,  123, -261,  265,  363,  579,  585,  612,  617,
+ /*   120 */   710,  754,  769,  774,  777,  781,  806,  807,  813,  820,
+ /*   130 */   830,  831,  832, -331, -289, -111, -249,  -61, -289,  290,
+ /*   140 */   563,  -25, -338,  158,  357, -147,  498,  533, -102,  202,
+ /*   150 */   324,   29,  378,  502,  593,  597,  808,  236, -367,  552,
+ /*   160 */   584,  603,  614,  674,  857,  614,  785,  871,  945,  877,
+ /*   170 */   811,  823,  937,  834,  920,  920,  946,  904,  963,  985,
+ /*   180 */   926,  919,  882,  882,  890,  882,  917,  911,  920,  949,
+ /*   190 */   957,  973,  996,  995,  999, 1002, 1051, 1052, 1010, 1013,
+ /*   200 */  1014, 1050, 1055, 1068, 1060, 1074, 1076, 1078, 1079, 1083,
+ /*   210 */  1088, 1085, 1017, 1077, 1046, 1080, 1090, 1035, 1087, 1091,
+ /*   220 */  1093, 1096, 1099, 1097, 1100, 1094, 1095, 1116, 1084, 1101,
+ /*   230 */  1102, 1103, 1107, 1108, 1111, 1112, 1114, 1115, 1122, 1098,
+ /*   240 */  1119, 1082, 1113, 1124, 1053, 1106, 1059, 1131, 1104, 1109,
+ /*   250 */  1123, 1148, 1118, 1158, 1120, 1127, 1126, 1054, 1125, 1128,
+ /*   260 */  1058, 1129, 1139, 1141,  920, 1069, 1073, 1081, 1075, 1086,
+ /*   270 */  1133, 1135, 1061, 1130, 1134,  882, 1198, 1144, 1201, 1200,
+ /*   280 */  1197, 1199, 1155, 1163, 1180, 1188, 1193, 1206, 1209, 1174,
+ /*   290 */  1210, 1173, 1220, 1227, 1236, 1239, 1150, 1221, 1214, 1237,
+ /*   300 */  1256, 1249, 1268, 1266, 1202, 1196, 1211, 1213, 1244, 1251,
+ /*   310 */  1252, 1262, 1281,
 };
 static const YYACTIONTYPE yy_default[] = {
  /*     0 */  1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719,
@@ -2454,8 +2478,8 @@ static const char *const yyRuleName[] = {
  /* 531 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP",
  /* 532 */ "fill_opt ::=",
  /* 533 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP",
- /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP",
- /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP",
+ /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP",
+ /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP",
  /* 536 */ "fill_mode ::= NONE",
  /* 537 */ "fill_mode ::= PREV",
  /* 538 */ "fill_mode ::= NULL",
@@ -3663,8 +3687,8 @@ static const struct {
   {  396,   -4 }, /* (531) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */
   {  456,    0 }, /* (532) fill_opt ::= */
   {  456,   -4 }, /* (533) fill_opt ::= FILL NK_LP fill_mode NK_RP */
-  {  456,   -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */
-  {  456,   -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */
+  {  456,   -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */
+  {  456,   -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */
   {  463,   -1 }, /* (536) fill_mode ::= NONE */
   {  463,   -1 }, /* (537) fill_mode ::= PREV */
   {  463,   -1 }, /* (538) fill_mode ::= NULL */
@@ -5351,10 +5375,10 @@ static YYACTIONTYPE yy_reduce(
       case 533: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */
 { yymsp[-3].minor.yy448 = createFillNode(pCxt, yymsp[-1].minor.yy46, NULL); }
         break;
-      case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */
+      case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */
 { yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); }
         break;
-      case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */
+      case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */
 { yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); }
         break;
       case 536: /* fill_mode ::= NONE */

From fdbb9cce48f66e8372c421cd9e28689da65f6e77 Mon Sep 17 00:00:00 2001
From: huolibo <huolibo@qq.com>
Date: Tue, 18 Apr 2023 16:08:22 +0800
Subject: [PATCH 58/65] fix: illegal accesses (#20960)

---
 source/client/src/clientJniConnector.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c
index d2a9665eee..b613354751 100644
--- a/source/client/src/clientJniConnector.c
+++ b/source/client/src/clientJniConnector.c
@@ -1259,8 +1259,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse
   int code = taos_errno(tres);
   if (code != TSDB_CODE_SUCCESS) {
     jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres));
+    jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres));
     taos_free_result(tres);
-    return createSchemalessResp(env, 0, code, taos_errstr(tres));
+    return jobject;
   }
   taos_free_result(tres);
 
@@ -1286,8 +1287,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse
   int code = taos_errno(tres);
   if (code != TSDB_CODE_SUCCESS) {
     jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres));
+    jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres));
     taos_free_result(tres);
-    return createSchemalessResp(env, 0, code, taos_errstr(tres));
+    return jobject;
   }
   taos_free_result(tres);
 
@@ -1315,8 +1317,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse
   int code = taos_errno(tres);
   if (code != TSDB_CODE_SUCCESS) {
     jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres));
+    jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres));
     taos_free_result(tres);
-    return createSchemalessResp(env, 0, code, taos_errstr(tres));
+    return jobject;
   }
   taos_free_result(tres);
 
@@ -1343,8 +1346,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse
   int code = taos_errno(tres);
   if (code != TSDB_CODE_SUCCESS) {
     jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres));
+    jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres));
     taos_free_result(tres);
-    return createSchemalessResp(env, 0, code, taos_errstr(tres));
+    return jobject;
   }
   taos_free_result(tres);
 

From 01c3df9a556fc550c269e1d30bf1764026a2eda6 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Tue, 18 Apr 2023 17:12:49 +0800
Subject: [PATCH 59/65] fix test cases

---
 tests/system-test/2-query/To_unixtimestamp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py
index 00a8ed84de..424ebff6c5 100644
--- a/tests/system-test/2-query/To_unixtimestamp.py
+++ b/tests/system-test/2-query/To_unixtimestamp.py
@@ -98,8 +98,8 @@ class TDTestCase:
     def timestamp_change_return_type(self):
         tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 0);")
         tdSql.checkEqual(tdSql.queryResult[0][0], 0)
-        tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1);")
-        tdSql.checkEqual(tdSql.queryResult[0][0], datetime.datetime(1970, 1, 1, 8, 0, 0))
+        tdSql.query(f"select to_unixtimestamp('1970-01-01 00:00:00', 1);")
+        tdSql.checkData(0, 0, '1970-01-01 00:00:00')
         tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 2);")
         tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1.5);")
         tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 'abc');")

From 6669c3a5ad392f242185f7a89b3fb2c2566ed766 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Tue, 18 Apr 2023 17:13:14 +0800
Subject: [PATCH 60/65] feat: last queries with tags output can be read from
 cache

---
 source/dnode/vnode/src/tsdb/tsdbCacheRead.c  | 17 +++++------
 source/libs/executor/src/cachescanoperator.c | 32 ++++++++++----------
 source/libs/planner/src/planOptimizer.c      | 21 ++++++++-----
 3 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
index 95981c2f08..f537b7d6e8 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
@@ -362,15 +362,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
               p->ts = pCol->ts;
               p->colVal = pCol->colVal;
               singleTableLastTs = pCol->ts;
-
-              // only set value for last row query
-              if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST_ROW)) {
-                if (taosArrayGetSize(pTableUidList) == 0) {
-                  taosArrayPush(pTableUidList, &pKeyInfo->uid);
-                } else {
-                  taosArraySet(pTableUidList, 0, &pKeyInfo->uid);
-                }
-              }
             }
           } else {
             SLastCol* p = taosArrayGet(pLastCols, slotId);
@@ -417,6 +408,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
         }
       }
 
+      if (0 == i) {
+        if (taosArrayGetSize(pTableUidList) == 0) {
+          taosArrayPush(pTableUidList, &pKeyInfo->uid);
+        } else {
+          taosArraySet(pTableUidList, 0, &pKeyInfo->uid);
+        }
+      }
+
       tsdbCacheRelease(lruCache, h);
     }
 
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index f6fc332b37..61f024ebb7 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -45,13 +45,13 @@ static void         destroyCacheScanOperator(void* param);
 static int32_t      extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds);
 static int32_t      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo);
 
-#define SCAN_ROW_TYPE(_t)  ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
+#define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
 
 SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
                                            STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
-  int32_t           code = TSDB_CODE_SUCCESS;
+  int32_t             code = TSDB_CODE_SUCCESS;
   SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo));
-  SOperatorInfo*    pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  SOperatorInfo*      pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     tableListDestroy(pTableListInfo);
@@ -91,7 +91,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
 
     uint64_t suid = tableListGetSuid(pTableListInfo);
     code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
-                                   taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str);
+                                   taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader,
+                                   pTaskInfo->id.str);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -114,7 +115,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
     p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset);
   }
 
-  setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo);
+  setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED,
+                  pInfo, pTaskInfo);
   pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
 
   pOperator->fpSet =
@@ -123,7 +125,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   pOperator->cost.openCost = 0;
   return pOperator;
 
-  _error:
+_error:
   pTaskInfo->code = code;
   destroyCacheScanOperator(pInfo);
   taosMemoryFree(pOperator);
@@ -136,8 +138,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
   }
 
   SCacheRowsScanInfo* pInfo = pOperator->info;
-  SExecTaskInfo*    pTaskInfo = pOperator->pTaskInfo;
-  STableListInfo*   pTableList = pInfo->pTableList;
+  SExecTaskInfo*      pTaskInfo = pOperator->pTaskInfo;
+  STableListInfo*     pTableList = pInfo->pTableList;
 
   uint64_t suid = tableListGetSuid(pTableList);
   int32_t  size = tableListGetSize(pTableList);
@@ -194,8 +196,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
       pRes->info.rows = 1;
 
       SExprSupp* pSup = &pInfo->pseudoExprSup;
-      int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes,
-                                            pRes->info.rows, GET_TASKID(pTaskInfo), NULL);
+      int32_t    code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes,
+                                               pRes->info.rows, GET_TASKID(pTaskInfo), NULL);
       if (code != TSDB_CODE_SUCCESS) {
         pTaskInfo->code = code;
         return NULL;
@@ -217,7 +219,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
       }
 
       STableKeyInfo* pList = NULL;
-      int32_t num = 0;
+      int32_t        num = 0;
 
       int32_t code = tableListGetGroupList(pTableList, pInfo->currentGroupIndex, &pList, &num);
       if (code != TSDB_CODE_SUCCESS) {
@@ -251,11 +253,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
           pInfo->pRes->info.id.groupId = pKeyInfo->groupId;
 
           if (taosArrayGetSize(pInfo->pUidList) > 0) {
-            ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW);
-
             pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0);
-            code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows,
-                                          GET_TASKID(pTaskInfo), NULL);
+            code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes,
+                                          pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), NULL);
             if (code != TSDB_CODE_SUCCESS) {
               pTaskInfo->code = code;
               return NULL;
@@ -325,7 +325,7 @@ int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pC
     return TSDB_CODE_SUCCESS;
   }
 
-  size_t size = taosArrayGetSize(pColMatchInfo->pList);
+  size_t  size = taosArrayGetSize(pColMatchInfo->pList);
   SArray* pMatchInfo = taosArrayInit(size, sizeof(SColMatchItem));
 
   for (int32_t i = 0; i < size; ++i) {
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 52bb03466c..d3a03b7b75 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -1095,7 +1095,7 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group
     *pNotOptimize = false;
     return TSDB_CODE_SUCCESS;
   }
-  
+
   switch (nodeType(pNode)) {
     case QUERY_NODE_LOGIC_PLAN_SCAN: {
       SScanLogicNode* pScan = (SScanLogicNode*)pNode;
@@ -2139,7 +2139,7 @@ typedef struct SLastRowScanOptLastParaCkCxt {
   bool hasCol;
 } SLastRowScanOptLastParaCkCxt;
 
-static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) {
+static EDealRes lastRowScanOptLastParaIsTagImpl(SNode* pNode, void* pContext) {
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
     SLastRowScanOptLastParaCkCxt* pCxt = pContext;
     if (COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType || COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType) {
@@ -2152,10 +2152,10 @@ static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-static bool lastRowScanOptLastParaCheck(SNode* pExpr) {
+static bool lastRowScanOptLastParaIsTag(SNode* pExpr) {
   SLastRowScanOptLastParaCkCxt cxt = {.hasTag = false, .hasCol = false};
-  nodesWalkExpr(pExpr, lastRowScanOptLastParaCheckImpl, &cxt);
-  return !cxt.hasTag && cxt.hasCol;
+  nodesWalkExpr(pExpr, lastRowScanOptLastParaIsTagImpl, &cxt);
+  return cxt.hasTag && !cxt.hasCol;
 }
 
 static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast) {
@@ -2195,15 +2195,19 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
   FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) {
     SFunctionNode* pAggFunc = (SFunctionNode*)pFunc;
     if (FUNCTION_TYPE_LAST == pAggFunc->funcType) {
-      if (hasSelectFunc || !lastRowScanOptLastParaCheck(nodesListGetNode(pAggFunc->pParameterList, 0))) {
+      if (hasSelectFunc) {
         return false;
       }
       hasLastFunc = true;
-    } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) {
+    } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) {
       if (hasLastFunc) {
         return false;
       }
       hasSelectFunc = true;
+    } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) {
+      if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) {
+        return false;
+      }
     } else if (FUNCTION_TYPE_LAST_ROW != pAggFunc->funcType) {
       return false;
     }
@@ -2282,6 +2286,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
         nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt);
         nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1));
       }
+    } else if (FUNCTION_TYPE_GROUP_KEY == funcType) {
+      nodesListMakeAppend(&cxt.pLastCols, nodesListGetNode(pFunc->pParameterList, 0));
     }
   }
 
@@ -2291,7 +2297,6 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
   if (NULL != cxt.pLastCols) {
     cxt.doAgg = false;
     lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols);
-    NODES_DESTORY_LIST(pScan->pScanPseudoCols);
     lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols);
     nodesClearList(cxt.pLastCols);
   }

From 1431a313b18813e50cddcc064e694e036736d26a Mon Sep 17 00:00:00 2001
From: sunpeng <sunpeng.dev@gmail.com>
Date: Tue, 18 Apr 2023 17:15:48 +0800
Subject: [PATCH 61/65] docs: fix for monitor doc (#20963)

---
 docs/zh/17-operation/10-monitor.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md
index b31ada1b79..918642c3d6 100644
--- a/docs/zh/17-operation/10-monitor.md
+++ b/docs/zh/17-operation/10-monitor.md
@@ -41,7 +41,7 @@ chmod +x TDinsight.sh
 
 ## log 库
 
-TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](..../reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。
+TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](/reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。
 
 ### cluster info 表
 

From 57e8c3c0915f7e9903592b0a40354a8ec3b1d796 Mon Sep 17 00:00:00 2001
From: sunpeng <sunpeng.dev@gmail.com>
Date: Tue, 18 Apr 2023 17:48:19 +0800
Subject: [PATCH 62/65] docs: update title for monitor doc (#20965)

---
 docs/zh/17-operation/10-monitor.md | 39 +++++++++++++++---------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md
index 918642c3d6..01a2257286 100644
--- a/docs/zh/17-operation/10-monitor.md
+++ b/docs/zh/17-operation/10-monitor.md
@@ -43,7 +43,7 @@ chmod +x TDinsight.sh
 
 TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](/reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。
 
-### cluster info 表
+### cluster\_info 表
 
 `cluster_info` 表记录集群信息。
 
@@ -72,7 +72,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |protocol|INT||协议版本，目前为 1|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### d info 表
+### d\_info 表
 
 `d_info` 表记录 dnode 状态信息。
 
@@ -83,7 +83,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### m info 表
+### m\_info 表
 
 `m_info` 表记录 mnode 角色信息。
 
@@ -95,7 +95,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |mnode\_ep|NCHAR|TAG|master node endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### dnodes info 表
+### dnodes\_info 表
 
 `dnodes_info` 记录 dnode 信息。
 
@@ -137,7 +137,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### data dir 表
+### data\_dir 表
 
 `data_dir` 表记录 data 目录信息。
 
@@ -153,7 +153,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### log dir 表
+### log\_dir 表
 
 `log_dir` 表记录 log 目录信息。
 
@@ -168,7 +168,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### tmp dir 表
+### temp\_dir 表
 
 `temp_dir` 表记录 temp 目录信息。
 
@@ -183,7 +183,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### vgroups info 表
+### vgroups\_info 表
 
 `vgroups_info` 表记录虚拟节点组信息。
 
@@ -198,7 +198,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### vnodes role 表
+### vnodes\_role 表
 
 `vnodes_role` 表记录虚拟节点角色信息。
 
@@ -223,7 +223,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### log summary 表
+### log\_summary 表
 
 `log_summary` 记录日志统计信息。
 
@@ -238,7 +238,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### grants info 表
+### grants\_info 表
 
 `grants_info` 记录授权信息。
 
@@ -252,7 +252,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |dnode\_ep|NCHAR|TAG|dnode endpoint|
 |cluster\_id|NCHAR|TAG|cluster id|
 
-### keeper monitor 表
+### keeper\_monitor 表
 
 `keeper_monitor` 记录 taoskeeper 监控数据。
 
@@ -263,7 +263,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |mem|FLOAT||内存使用率|
 |identify|NCHAR|TAG||
 
-### taosadapter restful http request total 表
+### taosadapter\_restful\_http\_request\_total 表
 
 `taosadapter_restful_http_request_total` 记录 taosadapter rest 请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
 
@@ -277,7 +277,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |request\_uri|NCHAR|TAG|request uri|
 |status\_code|NCHAR|TAG|status code|
 
-### taosadapter restful http request fail 表
+### taosadapter\_restful\_http\_request\_fail 表
 
 `taosadapter_restful_http_request_fail` 记录 taosadapter rest 请求失败信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
 
@@ -291,7 +291,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |request\_uri|NCHAR|TAG|request uri|
 |status\_code|NCHAR|TAG|status code|
 
-### taosadapter restful http request in flight 表
+### taosadapter\_restful\_http\_request\_in\_flight 表
 
 `taosadapter_restful_http_request_in_flight` 记录 taosadapter rest 实时请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
 
@@ -301,7 +301,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |guage|DOUBLE||监控指标值|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
 
-### taosadapter restful http request summary milliseconds 表
+### taosadapter\_restful\_http\_request\_summary\_milliseconds 表
 
 `taosadapter_restful_http_request_summary_milliseconds` 记录 taosadapter rest 请求汇总信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
 
@@ -319,7 +319,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |request\_method|NCHAR|TAG|request method|
 |request\_uri|NCHAR|TAG|request uri|
 
-### taosadapter system mem percent 表
+### taosadapter\_system\_mem\_percent 表
 
 `taosadapter_system_mem_percent` 表记录 taosadapter 内存使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
 
@@ -329,11 +329,12 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，
 |guage|DOUBLE||监控指标值|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
 
-### taosadapter system cpu percent 表
+### taosadapter\_system\_cpu\_percent 表
+
+`taosadapter_system_cpu_percent` 表记录 taosadapter cpu 使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。
 
 |field|type|is\_tag|comment|
 |:----|:---|:-----|:------|
 |\_ts|TIMESTAMP||timestamp|
 |guage|DOUBLE||监控指标值|
 |endpoint|NCHAR|TAG|taosadpater endpoint|
-

From 230adc5433dc6697e7abf11a7b4458cd39880b94 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjxilinx@users.noreply.github.com>
Date: Tue, 18 Apr 2023 18:38:17 +0800
Subject: [PATCH 63/65] Update 23-perf.md

---
 docs/zh/12-taos-sql/23-perf.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md
index d4ee0e178c..fc0b8072a7 100644
--- a/docs/zh/12-taos-sql/23-perf.md
+++ b/docs/zh/12-taos-sql/23-perf.md
@@ -69,7 +69,7 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其
 | 1   |  consumer_id   | BIGINT       | 消费者的唯一 ID                                             |
 | 2   | consumer_group | BINARY(192)  | 消费者组                                                    |
 | 3   |   client_id    | BINARY(192)  | 用户自定义字符串，通过创建 consumer 时指定 client_id 来展示 |
-| 4   |     status     | BINARY(20)   | 消费者当前状态                                              |
+| 4   |     status     | BINARY(20)   | 消费者当前状态。消费者状态包括：ready(正常可用)、 lost(连接已丢失)、 rebalancing(消费者所属 vgroup 正在分配中)、unknown(未知状态)|
 | 5   |     topics     | BINARY(204)  | 被订阅的 topic。若订阅多个 topic，则展示为多行              |
 | 6   |    up_time     | TIMESTAMP    | 第一次连接 taosd 的时间                                     |
 | 7   | subscribe_time | TIMESTAMP    | 上一次发起订阅的时间                                        |

From 18b4694327231cf086b3796495ce4be743bf9b7a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjxilinx@users.noreply.github.com>
Date: Tue, 18 Apr 2023 19:22:52 +0800
Subject: [PATCH 64/65] Update 23-perf.md

---
 docs/en/12-taos-sql/23-perf.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/12-taos-sql/23-perf.md b/docs/en/12-taos-sql/23-perf.md
index fc369ec663..43ff8e3091 100644
--- a/docs/en/12-taos-sql/23-perf.md
+++ b/docs/en/12-taos-sql/23-perf.md
@@ -69,7 +69,7 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE
 | 1   |  consumer_id   | BIGINT       | Consumer ID                                             |
 | 2   | consumer_group | BINARY(192)  | Consumer group                                                    |
 | 3   |   client_id    | BINARY(192)  | Client ID (user-defined) |
-| 4   |     status     | BINARY(20)   | Consumer status                                              |
+| 4   |     status     | BINARY(20)   | Consumer status. All possible status include: ready(consumer is in normal state), lost(the connection between consumer and mnode is broken), rebalance(the redistribution of vgroups that belongs to current consumer is now in progress), unknown(consumer is in invalid state)
 | 5   |     topics     | BINARY(204)  | Subscribed topic. Returns one row for each topic.              |
 | 6   |    up_time     | TIMESTAMP    | Time of first connection to TDengine Server                                     |
 | 7   | subscribe_time | TIMESTAMP    | Time of first subscription                                        |

From 586b3945d1571b2d0ca9320e80fbae147721849b Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Wed, 19 Apr 2023 13:09:11 +0800
Subject: [PATCH 65/65] feat: last queries with tags output can be read from
 cache

---
 source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 10 ++++------
 source/libs/planner/src/planOptimizer.c     | 12 +++++-------
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
index f537b7d6e8..d6e819d84a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
@@ -408,12 +408,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
         }
       }
 
-      if (0 == i) {
-        if (taosArrayGetSize(pTableUidList) == 0) {
-          taosArrayPush(pTableUidList, &pKeyInfo->uid);
-        } else {
-          taosArraySet(pTableUidList, 0, &pKeyInfo->uid);
-        }
+      if (taosArrayGetSize(pTableUidList) == 0) {
+        taosArrayPush(pTableUidList, &pKeyInfo->uid);
+      } else {
+        taosArraySet(pTableUidList, 0, &pKeyInfo->uid);
       }
 
       tsdbCacheRelease(lruCache, h);
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index d3a03b7b75..07ea110d7e 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -2195,7 +2195,7 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
   FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) {
     SFunctionNode* pAggFunc = (SFunctionNode*)pFunc;
     if (FUNCTION_TYPE_LAST == pAggFunc->funcType) {
-      if (hasSelectFunc) {
+      if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) {
         return false;
       }
       hasLastFunc = true;
@@ -2241,7 +2241,7 @@ static EDealRes lastRowScanOptSetColDataType(SNode* pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols) {
+static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, bool erase) {
   SNode* pTarget = NULL;
   WHERE_EACH(pTarget, pTargets) {
     bool   found = false;
@@ -2253,7 +2253,7 @@ static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCo
         break;
       }
     }
-    if (!found) {
+    if (!found && erase) {
       ERASE_NODE(pTargets);
       continue;
     }
@@ -2286,8 +2286,6 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
         nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt);
         nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1));
       }
-    } else if (FUNCTION_TYPE_GROUP_KEY == funcType) {
-      nodesListMakeAppend(&cxt.pLastCols, nodesListGetNode(pFunc->pParameterList, 0));
     }
   }
 
@@ -2296,8 +2294,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
   pScan->igLastNull = pAgg->hasLast ? true : false;
   if (NULL != cxt.pLastCols) {
     cxt.doAgg = false;
-    lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols);
-    lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols);
+    lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true);
+    lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false);
     nodesClearList(cxt.pLastCols);
   }
   pAgg->hasLastRow = false;