Merge remote-tracking branch 'origin/3.0' into feat/TD-25204

include/common/tglobal.h

@@ -195,7 +195,8 @@ extern int64_t tsWalFsyncDataSizeLimit;
 // internal
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;
-extern int32_t tsStreamCheckpointTickInterval;
+extern int32_t tsStreamCheckpointInterval;
+extern float   tsSinkDataRate;
 extern int32_t tsStreamNodeCheckInterval;
 extern int32_t tsTtlUnit;
 extern int32_t tsTtlPushIntervalSec;
@@ -204,9 +205,6 @@ extern int32_t tsTrimVDbIntervalSec;
 extern int32_t tsGrantHBInterval;
 extern int32_t tsUptimeInterval;
 
-extern int32_t tsRpcRetryLimit;
-extern int32_t tsRpcRetryInterval;
-
 extern bool    tsDisableStream;
 extern int64_t tsStreamBufferSize;
 extern bool    tsFilterScalarMode;

include/libs/stream/tstream.h

@@ -38,16 +38,25 @@ extern "C" {
 #define TASK_DOWNSTREAM_READY       0x0
 #define TASK_DOWNSTREAM_NOT_READY   0x1
 #define TASK_DOWNSTREAM_NOT_LEADER  0x2
-#define TASK_SELF_NEW_STAGE         0x3
+#define TASK_UPSTREAM_NEW_STAGE     0x3
 
 #define NODE_ROLE_UNINIT     0x1
 #define NODE_ROLE_LEADER     0x2
 #define NODE_ROLE_FOLLOWER   0x3
 
-typedef struct SStreamTask  SStreamTask;
+#define HAS_RELATED_FILLHISTORY_TASK(_t)  ((_t)->hTaskInfo.id.taskId != 0)
-typedef struct SStreamQueue SStreamQueue;
+#define CLEAR_RELATED_FILLHISTORY_TASK(_t) \
+  do {                                     \
+    (_t)->hTaskInfo.id.taskId = 0;         \
+    (_t)->hTaskInfo.id.streamId = 0;       \
+  } while (0)
+
+typedef struct SStreamTask   SStreamTask;
+typedef struct SStreamQueue  SStreamQueue;
+typedef struct SStreamTaskSM SStreamTaskSM;
 
 #define SSTREAM_TASK_VER 2
+
 enum {
   STREAM_STATUS__NORMAL = 0,
   STREAM_STATUS__STOP,
@@ -58,7 +67,7 @@ enum {
 };
 
 typedef enum ETaskStatus {
-  TASK_STATUS__NORMAL = 0,
+  TASK_STATUS__READY = 0,
   TASK_STATUS__DROPPING,
   TASK_STATUS__UNINIT,  // not used, an placeholder
   TASK_STATUS__STOP,
@@ -66,6 +75,7 @@ typedef enum ETaskStatus {
   TASK_STATUS__HALT,          // pause, but not be manipulated by user command
   TASK_STATUS__PAUSE,         // pause
   TASK_STATUS__CK,            // stream task is in checkpoint status, no data are allowed to put into inputQ anymore
+  TASK_STATUS__STREAM_SCAN_HISTORY,
 } ETaskStatus;
 
 enum {
@@ -118,6 +128,22 @@ enum {
   STREAM_META_OK_TO_STOP = 2,
 };
 
+typedef enum EStreamTaskEvent {
+  TASK_EVENT_INIT = 0x1,
+  TASK_EVENT_INIT_SCANHIST = 0x2,
+  TASK_EVENT_INIT_STREAM_SCANHIST = 0x3,
+  TASK_EVENT_SCANHIST_DONE = 0x4,
+  TASK_EVENT_STOP = 0x5,
+  TASK_EVENT_GEN_CHECKPOINT = 0x6,
+  TASK_EVENT_CHECKPOINT_DONE = 0x7,
+  TASK_EVENT_PAUSE = 0x8,
+  TASK_EVENT_RESUME = 0x9,
+  TASK_EVENT_HALT = 0xA,
+  TASK_EVENT_DROPPING = 0xB,
+  TASK_EVENT_SCAN_TSDB = 0xC,
+  TASK_EVENT_SCAN_WAL = 0xD,
+} EStreamTaskEvent;
+
 typedef struct {
   int8_t type;
 } SStreamQueueItem;
@@ -155,11 +181,6 @@ typedef struct {
   SSDataBlock* pBlock;
 } SStreamRefDataBlock;
 
-typedef struct {
-  int8_t       type;
-  SSDataBlock* pBlock;
-} SStreamTrigger;
-
 typedef struct SStreamQueueNode SStreamQueueNode;
 
 struct SStreamQueueNode {
@@ -215,6 +236,11 @@ typedef struct {
   SUseDbRsp dbInfo;
 } STaskDispatcherShuffle;
 
+typedef struct {
+  int32_t nodeId;
+  SEpSet  epset;
+} SDownstreamTaskEpset;
+
 typedef struct {
   int64_t         stbUid;
   char            stbFullName[TSDB_TABLE_FNAME_LEN];
@@ -265,14 +291,15 @@ typedef struct SCheckpointInfo {
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
-  int8_t  taskStatus;
+  SStreamTaskSM* pSM;
-  int8_t  downstreamReady;  // downstream tasks are all ready now, if this flag is set
+  int8_t         taskStatus;
-  int8_t  schedStatus;
+  int8_t         downstreamReady;  // downstream tasks are all ready now, if this flag is set
-  int8_t  keepTaskStatus;
+  int8_t         schedStatus;
-  bool    appendTranstateBlock;  // has append the transfer state data block already, todo: remove it
+  int8_t         keepTaskStatus;
-  int8_t  pauseAllowed;          // allowed task status to be set to be paused
+  bool           appendTranstateBlock;  // has append the transfer state data block already, todo: remove it
-  int32_t timerActive;           // timer is active
+  int8_t         pauseAllowed;          // allowed task status to be set to be paused
-  int32_t inScanHistorySentinel;
+  int32_t        timerActive;           // timer is active
+  int32_t        inScanHistorySentinel;
 } SStreamStatus;
 
 typedef struct SDataRange {
@@ -305,15 +332,10 @@ typedef struct SDispatchMsgInfo {
   void*   pTimer; // used to dispatch data after a given time duration
 } SDispatchMsgInfo;
 
-typedef struct STaskOutputQueue {
+typedef struct STaskQueue {
   int8_t        status;
   SStreamQueue* queue;
-} STaskOutputQueue;
+} STaskQueue;
-
-typedef struct STaskInputInfo {
-  int8_t        status;
-  SStreamQueue* queue;
-} STaskInputInfo;
 
 typedef struct STaskSchedInfo {
   int8_t  status;
@@ -349,6 +371,7 @@ typedef struct SHistoryTaskInfo {
   int32_t tickCount;
   int32_t retryTimes;
   int32_t waitInterval;
+  int64_t haltVer;      // offset in wal when halt the stream task
 } SHistoryTaskInfo;
 
 typedef struct STaskOutputInfo {
@@ -361,6 +384,7 @@ typedef struct STaskOutputInfo {
   };
   int8_t        type;
   STokenBucket* pTokenBucket;
+  SArray*       pDownstreamUpdateList;
 } STaskOutputInfo;
 
 typedef struct SUpstreamInfo {
@@ -372,8 +396,8 @@ struct SStreamTask {
   int64_t             ver;
   SStreamTaskId       id;
   SSTaskBasicInfo     info;
-  STaskOutputQueue    outputq;
+  STaskQueue          outputq;
-  STaskInputInfo      inputInfo;
+  STaskQueue          inputq;
   STaskSchedInfo      schedInfo;
   STaskOutputInfo     outputInfo;
   SDispatchMsgInfo    msgInfo;
@@ -408,38 +432,43 @@ struct SStreamTask {
 typedef struct STaskStartInfo {
   int64_t   startTs;
   int64_t   readyTs;
-  int32_t   startedAfterNodeUpdate;
+  int32_t   startAllTasksFlag;
   SHashObj* pReadyTaskSet;           // tasks that are all ready for running stream processing
   int32_t   elapsedTime;
 } STaskStartInfo;
 
+typedef struct STaskUpdateInfo {
+  SHashObj* pTasks;
+  int32_t   transId;
+} STaskUpdateInfo;
+
 // meta
 typedef struct SStreamMeta {
-  char*         path;
+  char*           path;
-  TDB*          db;
+  TDB*            db;
-  TTB*          pTaskDb;
+  TTB*            pTaskDb;
-  TTB*          pCheckpointDb;
+  TTB*            pCheckpointDb;
-  SHashObj*     pTasksMap;
+  SHashObj*       pTasksMap;
-  SArray*       pTaskList;  // SArray<STaskId*>
+  SArray*         pTaskList;  // SArray<STaskId*>
-  void*         ahandle;
+  void*           ahandle;
-  TXN*          txn;
+  TXN*            txn;
-  FTaskExpand*  expandFunc;
+  FTaskExpand*    expandFunc;
-  int32_t       vgId;
+  int32_t         vgId;
-  int64_t       stage;
+  int64_t         stage;
-  int32_t       role;
+  int32_t         role;
-  STaskStartInfo startInfo;
+  STaskStartInfo  startInfo;
-  SRWLatch      lock;
+  SRWLatch        lock;
-  int32_t       walScanCounter;
+  int32_t         walScanCounter;
-  void*         streamBackend;
+  void*           streamBackend;
-  int64_t       streamBackendRid;
+  int64_t         streamBackendRid;
-  SHashObj*     pTaskBackendUnique;
+  SHashObj*       pTaskBackendUnique;
-  TdThreadMutex backendMutex;
+  TdThreadMutex   backendMutex;
-  SMetaHbInfo*  pHbInfo;
+  SMetaHbInfo*    pHbInfo;
-  SHashObj*     pUpdateTaskSet;
+  STaskUpdateInfo updateInfo;
-  int32_t       numOfStreamTasks;  // this value should be increased when a new task is added into the meta
+  int32_t         numOfStreamTasks;  // this value should be increased when a new task is added into the meta
-  int32_t       numOfPausedTasks;
+  int32_t         numOfPausedTasks;
-  int32_t       chkptNotReadyTasks;
+  int32_t         chkptNotReadyTasks;
-  int64_t       rid;
+  int64_t         rid;
 
   int64_t  chkpId;
   SArray*  chkpSaved;
@@ -617,7 +646,8 @@ typedef struct STaskStatusEntry {
 typedef struct SStreamHbMsg {
   int32_t vgId;
   int32_t numOfTasks;
-  SArray* pTaskStatus;  // SArray<SStreamTaskStatusEntry>
+  SArray* pTaskStatus;  // SArray<STaskStatusEntry>
+  SArray* pUpdateNodes; // SArray<int32_t>, needs update the epsets in stream tasks for those nodes.
 } SStreamHbMsg;
 
 int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp);
@@ -641,6 +671,7 @@ typedef struct SNodeUpdateInfo {
 } SNodeUpdateInfo;
 
 typedef struct SStreamTaskNodeUpdateMsg {
+  int32_t transId;   // to identify the msg
   int64_t streamId;
   int32_t taskId;
   SArray* pNodeList;  // SArray<SNodeUpdateInfo>
@@ -686,24 +717,36 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t
 void    streamTaskInputFail(SStreamTask* pTask);
 int32_t streamExecTask(SStreamTask* pTask);
 int32_t streamSchedExec(SStreamTask* pTask);
-bool    streamTaskShouldStop(const SStreamStatus* pStatus);
+bool    streamTaskShouldStop(const SStreamTask* pStatus);
-bool    streamTaskShouldPause(const SStreamStatus* pStatus);
+bool    streamTaskShouldPause(const SStreamTask* pStatus);
 bool    streamTaskIsIdle(const SStreamTask* pTask);
+bool    streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
+
+char*       createStreamTaskIdStr(int64_t streamId, int32_t taskId);
+ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr);
+const char* streamTaskGetStatusStr(ETaskStatus status);
+void        streamTaskResetStatus(SStreamTask* pTask);
+void        streamTaskSetStatusReady(SStreamTask* pTask);
 
 void    initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);
 
-char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
-
 // recover and fill history
 void    streamTaskCheckDownstream(SStreamTask* pTask);
-int32_t streamTaskStartScanHistory(SStreamTask* pTask);
+int32_t onNormalTaskReady(SStreamTask* pTask);
+int32_t onScanhistoryTaskReady(SStreamTask* pTask);
+
 int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage);
 int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
 void    streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
 bool    streamTaskAllUpstreamClosed(SStreamTask* pTask);
 bool    streamTaskSetSchedStatusWait(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusActive(SStreamTask* pTask);
-int8_t  streamTaskSetSchedStatusInActive(SStreamTask* pTask);
+int8_t  streamTaskSetSchedStatusInactive(SStreamTask* pTask);
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask);
+
+int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
+void    streamTaskRestoreStatus(SStreamTask* pTask);
 
 int32_t streamTaskStop(SStreamTask* pTask);
 int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp,
@@ -714,16 +757,11 @@ int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask);
 int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
 bool    streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
-int32_t streamQueueGetAvailableSpace(const SStreamQueue* pQueue, int32_t* availNum, double* availSize);
 
 // common
 int32_t     streamRestoreParam(SStreamTask* pTask);
-int32_t     streamSetStatusNormal(SStreamTask* pTask);
-int32_t     streamSetStatusUnint(SStreamTask* pTask);
-const char* streamGetTaskStatusStr(int32_t status);
 void        streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta);
-void        streamTaskResume(SStreamTask* pTask, SStreamMeta* pMeta);
+void        streamTaskResume(SStreamTask* pTask);
-void        streamTaskResumeFromHalt(SStreamTask* pTask);
 void        streamTaskDisablePause(SStreamTask* pTask);
 void        streamTaskEnablePause(SStreamTask* pTask);
 int32_t     streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask);
@@ -759,7 +797,6 @@ int32_t      streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
 int32_t      streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
 int32_t      streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);
-int32_t      streamMetaGetNumOfStreamTasks(SStreamMeta* pMeta);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaReopen(SStreamMeta* pMeta);
@@ -768,12 +805,13 @@ int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
 void         streamMetaStartHb(SStreamMeta* pMeta);
 void         streamMetaInitForSnode(SStreamMeta* pMeta);
+bool         streamMetaTaskInTimer(SStreamMeta* pMeta);
+int32_t      streamMetaUpdateTaskReadyInfo(SStreamTask* pTask);
 
 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask);
-
 int32_t streamAlignTransferState(SStreamTask* pTask);
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask,

include/util/taoserror.h

@@ -332,6 +332,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_INVALID_REPLICA           TAOS_DEF_ERROR_CODE(0, 0x03B7)
 #define TSDB_CODE_MND_DNODE_IN_CREATING         TAOS_DEF_ERROR_CODE(0, 0x03B8)
 #define TSDB_CODE_MND_DNODE_IN_DROPPING         TAOS_DEF_ERROR_CODE(0, 0x03B9)
+#define TSDB_CODE_MND_NO_ENOUGH_VNODES          TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-stable-part2
 #define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC  TAOS_DEF_ERROR_CODE(0, 0x03C0)

include/util/tconfig.h

@@ -65,7 +65,6 @@ typedef struct SConfigItem {
   union {
     bool    bval;
     float   fval;
-    double  dval;
     int32_t i32;
     int64_t i64;
     char   *str;
@@ -104,7 +103,6 @@ int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scop
 int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
 int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
 int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope);
-int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope);
 int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
 int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
 int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);

include/util/tqueue.h

@@ -101,6 +101,9 @@ struct STaosQall {
   STaosQnode *current;
   STaosQnode *start;
   int32_t     numOfItems;
+  int64_t     memOfItems;
+  int32_t     unAccessedNumOfItems;
+  int64_t     unAccessMemOfItems;
 };
 
 STaosQueue *taosOpenQueue();
@@ -123,6 +126,9 @@ int32_t    taosReadAllQitems(STaosQueue *queue, STaosQall *qall);
 int32_t    taosGetQitem(STaosQall *qall, void **ppItem);
 void       taosResetQitems(STaosQall *qall);
 int32_t    taosQallItemSize(STaosQall *qall);
+int64_t    taosQallMemSize(STaosQall *qll);
+int64_t    taosQallUnAccessedItemSize(STaosQall *qall);
+int64_t    taosQallUnAccessedMemSize(STaosQall *qall);
 
 STaosQset *taosOpenQset();
 void       taosCloseQset(STaosQset *qset);
@@ -135,8 +141,6 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo)
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
 
-extern int64_t tsRpcQueueMemoryAllowed;
-
 #ifdef __cplusplus
 }
 #endif

source/common/src/tglobal.c

@@ -254,7 +254,8 @@ int32_t tsTtlBatchDropNum = 10000;   // number of tables dropped per batch
 // internal
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
-int32_t tsStreamCheckpointTickInterval = 300;
+int32_t tsStreamCheckpointInterval = 60;
+float   tsSinkDataRate = 2.0;
 int32_t tsStreamNodeCheckInterval = 30;
 int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushIntervalSec = 10;
@@ -282,8 +283,6 @@ char   tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
 int32_t tsS3BlockSize = 4096;     // number of tsdb pages
 int32_t tsS3BlockCacheSize = 16;  // number of blocks
 
-int32_t tsCheckpointInterval = 300;
-
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
   SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@@ -512,11 +511,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1;
   if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "AVX", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "FMA", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1;
   if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1;
 
   if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1;
@@ -673,18 +672,18 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointTickInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER) != 0) return -1;
 
-  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0)
+  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1;
-    return -1;
 
-  if (cfgAddString(pCfg, "LossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddFloat(pCfg, "FPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddDouble(pCfg, "DPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "MaxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "CurRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddBool(pCfg, "IfAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddString(pCfg, "Compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
 
   if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
@@ -1046,7 +1045,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
   tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
 
-  tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "SIMD-builtins")->bval;
+  tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "simdEnable")->bval;
   tsTagFilterCache = (bool)cfgGetItem(pCfg, "tagFilterCache")->bval;
 
   tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
@@ -1105,16 +1104,18 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
 
-  tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "LossyColumns")->str, sizeof(tsLossyColumns));
+  tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "lossyColumns")->str, sizeof(tsLossyColumns));
-  tsFPrecision = cfgGetItem(pCfg, "FPrecision")->fval;
+  tsFPrecision = cfgGetItem(pCfg, "fPrecision")->fval;
-  tsDPrecision = cfgGetItem(pCfg, "DPrecision")->dval;
+  tsDPrecision = cfgGetItem(pCfg, "dPrecision")->fval;
-  tsMaxRange = cfgGetItem(pCfg, "MaxRange")->i32;
+  tsMaxRange = cfgGetItem(pCfg, "maxRange")->i32;
-  tsCurRange = cfgGetItem(pCfg, "CurRange")->i32;
+  tsCurRange = cfgGetItem(pCfg, "curRange")->i32;
-  tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
+  tsIfAdtFse = cfgGetItem(pCfg, "ifAdtFse")->bval;
-  tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
+  tstrncpy(tsCompressor, cfgGetItem(pCfg, "compressor")->str, sizeof(tsCompressor));
 
   tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
   tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
+  tsStreamCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i32;
+  tsSinkDataRate = cfgGetItem(pCfg, "streamSinkDataRate")->fval;
 
   tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval;
   tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32;

source/common/src/tmsg.c

@@ -8574,7 +8574,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
       } else {
         tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE);
       }
-      taosMemoryFree(pTbData->pCreateTbReq);
+      taosMemoryFreeClear(pTbData->pCreateTbReq);
     }
 
     if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {

source/dnode/mnode/impl/src/mndMain.c

@@ -282,7 +282,7 @@ static void *mndThreadFp(void *param) {
       mndCalMqRebalance(pMnode);
     }
 
-    if (sec % tsStreamCheckpointTickInterval == 0) {
+    if (sec % tsStreamCheckpointInterval == 0) {
       mndStreamCheckpointTick(pMnode, sec);
     }
 

source/dnode/mnode/impl/src/mndScheduler.c

@@ -569,6 +569,10 @@ static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStr
 }
 
 static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) {
+  if (taosArrayGetSize(pTasksList) < SINK_NODE_LEVEL || pUpstreamTask == NULL) {
+    return;
+  }
+
   SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL);
   for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) {
     SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i);
@@ -628,7 +632,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
     }
 
     setSinkTaskUpstreamInfo(pStream->tasks, pAggTask);
-    setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask);
+    if (pHAggTask != NULL) {
+      setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask);
+    }
 
     // source level
     return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, nextWindowSkey);

source/dnode/mnode/impl/src/mndStream.c

@@ -42,22 +42,22 @@ typedef struct SNodeEntry {
   int64_t hbTimestamp;  // second
 } SNodeEntry;
 
-typedef struct SStreamExecNodeInfo {
+typedef struct SStreamExecInfo {
   SArray       *pNodeEntryList;
   int64_t       ts;                // snapshot ts
   int64_t       activeCheckpoint;  // active check point id
   SHashObj     *pTaskMap;
   SArray       *pTaskList;
   TdThreadMutex lock;
-} SStreamExecNodeInfo;
+} SStreamExecInfo;
 
 typedef struct SVgroupChangeInfo {
   SHashObj *pDBMap;
   SArray   *pUpdateNodeList;  // SArray<SNodeUpdateInfo>
 } SVgroupChangeInfo;
 
-static int32_t                 mndNodeCheckSentinel = 0;
+static int32_t         mndNodeCheckSentinel = 0;
-static SStreamExecNodeInfo     execNodeList;
+static SStreamExecInfo execInfo;
 
 static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
@@ -77,19 +77,21 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
                                                   int64_t streamId, int32_t taskId);
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
 static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
+static SArray *extractNodeListFromStream(SMnode *pMnode);
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady);
 
-static SArray           *extractNodeListFromStream(SMnode *pMnode);
-static SArray           *mndTakeVgroupSnapshot(SMnode *pMnode);
 static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
 
 static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name);
 static int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans);
-static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset);
+static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
-static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo);
+                            int32_t retryCode);
-
+static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
-static void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode);
+static void    removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
-static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode);
+static void    keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
 static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot);
+static int32_t doKillActiveCheckpointTrans(SMnode *pMnode);
+static int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList);
 
 int32_t mndInitStream(SMnode *pMnode) {
   SSdbTable table = {
@@ -129,18 +131,18 @@ int32_t mndInitStream(SMnode *pMnode) {
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);
 
-  taosThreadMutexInit(&execNodeList.lock, NULL);
+  taosThreadMutexInit(&execInfo.lock, NULL);
-  execNodeList.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
+  execInfo.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
-  execNodeList.pTaskList = taosArrayInit(4, sizeof(STaskId));
+  execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
 
   return sdbSetTable(pMnode->pSdb, table);
 }
 
 void mndCleanupStream(SMnode *pMnode) {
-  taosArrayDestroy(execNodeList.pTaskList);
+  taosArrayDestroy(execInfo.pTaskList);
-  taosHashCleanup(execNodeList.pTaskMap);
+  taosHashCleanup(execInfo.pTaskMap);
-  taosThreadMutexDestroy(&execNodeList.lock);
+  taosThreadMutexDestroy(&execInfo.lock);
-  mDebug("mnd stream cleanup");
+  mDebug("mnd stream exec info cleanup");
 }
 
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@@ -286,7 +288,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
 static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
   int8_t status = atomic_load_8(&pStream->status);
   if (status == STREAM_STATUS__NORMAL) {
-    strcpy(dst, "normal");
+    strcpy(dst, "ready");
   } else if (status == STREAM_STATUS__STOP) {
     strcpy(dst, "stop");
   } else if (status == STREAM_STATUS__FAILED) {
@@ -294,7 +296,7 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
   } else if (status == STREAM_STATUS__RECOVER) {
     strcpy(dst, "recover");
   } else if (status == STREAM_STATUS__PAUSE) {
-    strcpy(dst, "pause");
+    strcpy(dst, "paused");
   }
 }
 
@@ -516,7 +518,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
 
   STransAction action = {0};
   action.mTraceId = pTrans->mTraceId;
-  initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet);
+  initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(buf);
     return -1;
@@ -688,7 +690,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
   pReq->streamId = pTask->id.streamId;
 
   STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);
     return -1;
@@ -847,10 +849,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
 
   mndTransDrop(pTrans);
 
-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
   mDebug("register to stream task node list");
-  keepStreamTasksInBuf(&streamObj, &execNodeList);
+  keepStreamTasksInBuf(&streamObj, &execInfo);
-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);
 
   code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -882,9 +884,8 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
     return 0;
   }
 
-  int64_t                  checkpointId = taosGetTimestampMs();
   SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
-  pMsg->checkpointId = checkpointId;
+  pMsg->checkpointId = taosGetTimestampMs();
 
   int32_t size = sizeof(SMStreamDoCheckpointMsg);
   SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@@ -936,7 +937,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
 }
 // static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) {
 //   int64_t timestampMs = taosGetTimestampMs();
-//   if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointTickInterval * 1000) {
+//   if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000) {
 //     return -1;
 //   }
 
@@ -1069,7 +1070,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
 
         STransAction action = {0};
         SEpSet       epset = mndGetVgroupEpset(pMnode, pVgObj);
-        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset);
+        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
         mndReleaseVgroup(pMnode, pVgObj);
 
         if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@@ -1084,6 +1085,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
   pStream->checkpointId = chkptId;
   pStream->checkpointFreq = taosGetTimestampMs();
   pStream->currentTick = 0;
+
   // 3. commit log: stream checkpoint info
   pStream->version = pStream->version + 1;
 
@@ -1133,31 +1135,37 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
   {  // check if the node update happens or not
     int64_t ts = taosGetTimestampSec();
 
-    if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) {
+    if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
-      if (execNodeList.pNodeEntryList != NULL) {
+      if (execInfo.pNodeEntryList != NULL) {
-        execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+        execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
       }
 
-      execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode);
+      execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
     }
 
-    if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) {
+    if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
       mDebug("stream task node change checking done, no vgroups exist, do nothing");
-      execNodeList.ts = ts;
+      execInfo.ts = ts;
       return 0;
     }
 
-    for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) {
+    for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
-      SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, i);
+      SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
       if (pNodeEntry->stageUpdated) {
         mDebug("stream task not ready due to node update detected, checkpoint not issued");
         return 0;
       }
     }
 
-    SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode);
+    bool allReady = true;
+    SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
+    if (!allReady) {
+      mWarn("not all vnodes are ready, ignore the checkpoint")
+      taosArrayDestroy(pNodeSnapshot);
+      return 0;
+    }
 
-    SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot);
+    SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
     bool              nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0);
     taosArrayDestroy(changeInfo.pUpdateNodeList);
     taosHashCleanup(changeInfo.pDBMap);
@@ -1169,26 +1177,25 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
     }
   }
 
-  {  // check if all tasks are in TASK_STATUS__NORMAL status
+  {  // check if all tasks are in TASK_STATUS__READY status
     bool ready = true;
 
-    taosThreadMutexLock(&execNodeList.lock);
+    taosThreadMutexLock(&execInfo.lock);
-    for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pTaskList); ++i) {
+    for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
-      STaskId *p = taosArrayGet(execNodeList.pTaskList, i);
+      STaskId *p = taosArrayGet(execInfo.pTaskList, i);
-      STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, p, sizeof(*p));
+      STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
       if (pEntry == NULL) {
         continue;
       }
 
-      if (pEntry->status != TASK_STATUS__NORMAL) {
+      if (pEntry->status != TASK_STATUS__READY) {
         mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued",
-              pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamGetTaskStatusStr(pEntry->status));
+              pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status));
         ready = false;
         break;
       }
     }
-    taosThreadMutexUnlock(&execNodeList.lock);
+    taosThreadMutexUnlock(&execInfo.lock);
-
     if (!ready) {
       return 0;
     }
@@ -1202,7 +1209,8 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
     mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
     return -1;
   }
-  mDebug("start to trigger checkpoint, checkpointId: %" PRId64 "", checkpointId);
+
+  mDebug("start to trigger checkpoint, checkpointId: %" PRId64, checkpointId);
 
   const char *pDb = mndGetStreamDB(pMnode);
   mndTransSetDbName(pTrans, pDb, "checkpoint");
@@ -1228,11 +1236,16 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
 
   if (code == 0) {
     if (mndTransPrepare(pMnode, pTrans) != 0) {
-      mError("failed to prepre trans rebalance since %s", terrstr());
+      mError("failed to prepare trans rebalance since %s", terrstr());
     }
   }
 
   mndTransDrop(pTrans);
+
+  // only one trans here
+  taosThreadMutexLock(&execInfo.lock);
+  execInfo.activeCheckpoint = checkpointId;
+  taosThreadMutexUnlock(&execInfo.lock);
   return code;
 }
 
@@ -1310,7 +1323,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
     return -1;
   }
 
-  removeStreamTasksInBuf(pStream, &execNodeList);
+  removeStreamTasksInBuf(pStream, &execInfo);
 
   SName name = {0};
   tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB);
@@ -1561,12 +1574,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         char status[20 + VARSTR_HEADER_SIZE] = {0};
 
         STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-        STaskStatusEntry* pe = taosHashGet(execNodeList.pTaskMap, &id, sizeof(id));
+        STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
         if (pe == NULL) {
           continue;
         }
 
-        const char* pStatus = streamGetTaskStatusStr(pe->status);
+        const char* pStatus = streamTaskGetStatusStr(pe->status);
         STR_TO_VARSTR(status, pStatus);
 
         // status
@@ -1641,7 +1654,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) {
   pReq->streamId = pTask->id.streamId;
 
   STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);
     return -1;
@@ -1774,7 +1787,7 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t ig
   pReq->igUntreated = igUntreated;
 
   STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);
     return -1;
@@ -1878,18 +1891,19 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
   return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
-static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, int64_t streamId,
+static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
-                              int32_t taskId) {
+                              int32_t transId) {
-  pMsg->streamId = streamId;
+  pMsg->streamId = pId->streamId;
-  pMsg->taskId = taskId;
+  pMsg->taskId = pId->taskId;
+  pMsg->transId = transId;
   pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
   taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
 }
 
 static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
-                                          int64_t streamId, int32_t taskId) {
+                                          SStreamTaskId* pId, int32_t transId) {
   SStreamTaskNodeUpdateMsg req = {0};
-  initNodeUpdateMsg(&req, pInfo, streamId, taskId);
+  initNodeUpdateMsg(&req, pInfo, pId, transId);
 
   int32_t code = 0;
   int32_t blen;
@@ -1953,20 +1967,19 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans) {
   return 0;
 }
 
-void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset) {
+void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
+                     int32_t retryCode) {
   pAction->epSet = *pEpset;
   pAction->contLen = contLen;
   pAction->pCont = pCont;
   pAction->msgType = msgType;
+  pAction->retryCode = retryCode;
 }
 
 // todo extract method: traverse stream tasks
 // build trans to update the epset
-static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo) {
+static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
-  STrans* pTrans = doCreateTrans(pMnode, pStream, "stream-task-update");
+  mDebug("start to build stream:0x%" PRIx64 " tasks epset update", pStream->uid);
-  if (pTrans == NULL) {
-    return terrno;
-  }
 
   taosWLockLatch(&pStream->lock);
   int32_t numOfLevels = taosArrayGetSize(pStream->tasks);
@@ -1981,38 +1994,20 @@ static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgr
       void   *pBuf = NULL;
       int32_t len = 0;
       streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
-      doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId);
+      doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
 
       STransAction action = {0};
-      initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet);
+      initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
       if (mndTransAppendRedoAction(pTrans, &action) != 0) {
         taosMemoryFree(pBuf);
         taosWUnLockLatch(&pStream->lock);
-        mndTransDrop(pTrans);
         return -1;
       }
     }
   }
 
   taosWUnLockLatch(&pStream->lock);
-
+  return 0;
-  int32_t code = mndPersistTransLog(pStream, pTrans);
-  if (code != TSDB_CODE_SUCCESS) {
-    sdbRelease(pMnode->pSdb, pStream);
-    return -1;
-  }
-
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
-    sdbRelease(pMnode->pSdb, pStream);
-    mndTransDrop(pTrans);
-    return -1;
-  }
-
-  sdbRelease(pMnode->pSdb, pStream);
-  mndTransDrop(pTrans);
-
-  return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
 static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) {
@@ -2050,8 +2045,8 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
 
           char buf[256] = {0};
           EPSET_TO_STR(&pCurrent->epset, buf);
-          mDebug("nodeId:%d epset changed detected, old:%s:%d -> new:%s", pCurrent->nodeId, pPrevEp->fqdn,
+          mDebug("nodeId:%d restart/epset changed detected, old:%s:%d -> new:%s, stageUpdate:%d", pCurrent->nodeId,
-                 pPrevEp->port, buf);
+                 pPrevEp->fqdn, pPrevEp->port, buf, pPrevEntry->stageUpdated);
 
           SNodeUpdateInfo updateInfo = {.nodeId = pPrevEntry->nodeId};
           epsetAssign(&updateInfo.prevEp, &pPrevEntry->epset);
@@ -2071,11 +2066,12 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
   return info;
 }
 
-static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) {
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) {
   SSdb   *pSdb = pMnode->pSdb;
   void   *pIter = NULL;
   SVgObj *pVgroup = NULL;
 
+  *allReady = true;
   SArray *pVgroupListSnapshot = taosArrayInit(4, sizeof(SNodeEntry));
 
   while (1) {
@@ -2087,8 +2083,26 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) {
     SNodeEntry entry = {0};
     entry.epset = mndGetVgroupEpset(pMnode, pVgroup);
     entry.nodeId = pVgroup->vgId;
-    entry.hbTimestamp = -1;
+    entry.hbTimestamp = pVgroup->updateTime;
 
+    if (*allReady) {
+      for (int32_t i = 0; i < pVgroup->replica; ++i) {
+        if (!pVgroup->vnodeGid[i].syncRestore) {
+          *allReady = false;
+          break;
+        }
+
+        ESyncState state = pVgroup->vnodeGid[i].syncState;
+        if (state == TAOS_SYNC_STATE_OFFLINE || state == TAOS_SYNC_STATE_ERROR) {
+          *allReady = false;
+          break;
+        }
+      }
+    }
+
+    char buf[256] = {0};
+    EPSET_TO_STR(&entry.epset, buf);
+    mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
     taosArrayPush(pVgroupListSnapshot, &entry);
     sdbRelease(pSdb, pVgroup);
   }
@@ -2102,27 +2116,63 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
   // check all streams that involved this vnode should update the epset info
   SStreamObj *pStream = NULL;
   void       *pIter = NULL;
+  STrans     *pTrans = NULL;
+
   while (1) {
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
     if (pIter == NULL) {
       break;
     }
 
+    // here create only one trans
+    if (pTrans == NULL) {
+      pTrans = doCreateTrans(pMnode, pStream, "stream-task-update");
+      if (pTrans == NULL) {
+        sdbRelease(pSdb, pStream);
+        sdbCancelFetch(pSdb, pIter);
+        return terrno;
+      }
+    }
+
     void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb));
     void *p1 = taosHashGet(pChangeInfo->pDBMap, pStream->sourceDb, strlen(pStream->sourceDb));
     if (p == NULL && p1 == NULL) {
-      mndReleaseStream(pMnode, pStream);
+      mDebug("stream:0x%" PRIx64 " %s not involved nodeUpdate, ignore", pStream->uid, pStream->name);
+      sdbRelease(pSdb, pStream);
       continue;
     }
 
-    mDebug("stream:0x%" PRIx64 " involved node changed, create update trans", pStream->uid);
+    mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
-    int32_t code = createStreamUpdateTrans(pMnode, pStream, pChangeInfo);
+           pStream->name, pTrans->id);
+
+    int32_t code = createStreamUpdateTrans(pStream, pChangeInfo, pTrans);
+
+    // todo: not continue, drop all and retry again
+    if (code != TSDB_CODE_SUCCESS) {
+      mError("stream:0x%" PRIx64 " build nodeUpdate trans failed, ignore and continue, code:%s", pStream->uid,
+             tstrerror(code));
+      sdbRelease(pSdb, pStream);
+      continue;
+    }
+
+    code = mndPersistTransLog(pStream, pTrans);
+    sdbRelease(pSdb, pStream);
+
     if (code != TSDB_CODE_SUCCESS) {
       sdbCancelFetch(pSdb, pIter);
-      return code;
+      return -1;
     }
   }
 
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
+    sdbRelease(pMnode->pSdb, pStream);
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  sdbRelease(pMnode->pSdb, pStream);
+  mndTransDrop(pTrans);
   return 0;
 }
 
@@ -2165,6 +2215,10 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
   while ((pIter = taosHashIterate(pHash, pIter)) != NULL) {
     SNodeEntry *pEntry = (SNodeEntry *)pIter;
     taosArrayPush(plist, pEntry);
+
+    char buf[256] = {0};
+    EPSET_TO_STR(&pEntry->epset, buf);
+    mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf);
   }
   taosHashCleanup(pHash);
 
@@ -2182,28 +2236,31 @@ static void doExtractTasksFromStream(SMnode *pMnode) {
       break;
     }
 
-    keepStreamTasksInBuf(pStream, &execNodeList);
+    keepStreamTasksInBuf(pStream, &execInfo);
     sdbRelease(pSdb, pStream);
   }
 }
 
-static int32_t doRemoveFromTask(SStreamExecNodeInfo* pExecNode, STaskId* pRemovedId) {
+static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
   void *p = taosHashGet(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
+  if (p == NULL) {
+    return TSDB_CODE_SUCCESS;
+  }
 
-  if (p != NULL) {
+  taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
-    taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
 
-    for(int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
+  for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
-      STaskId* pId = taosArrayGet(pExecNode->pTaskList, k);
+    STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
-      if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) {
+    if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) {
-        taosArrayRemove(pExecNode->pTaskList, k);
+      taosArrayRemove(pExecNode->pTaskList, k);
-        mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t) pRemovedId->taskId,
+
-              (int32_t)taosArrayGetSize(pExecNode->pTaskList));
+      int32_t num = taosArrayGetSize(pExecNode->pTaskList);
-        break;
+      mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)pRemovedId->taskId, num);
-      }
+      break;
     }
   }
-  return 0;
+
+  return TSDB_CODE_SUCCESS;
 }
 
 static bool taskNodeExists(SArray* pList, int32_t nodeId) {
@@ -2220,31 +2277,31 @@ static bool taskNodeExists(SArray* pList, int32_t nodeId) {
 }
 
 int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
-  SArray* pRemoveTaskList = taosArrayInit(4, sizeof(STaskId));
+  SArray* pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
 
-  int32_t numOfTask = taosArrayGetSize(execNodeList.pTaskList);
+  int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
   for(int32_t i = 0; i < numOfTask; ++i) {
-    STaskId* pId = taosArrayGet(execNodeList.pTaskList, i);
+    STaskId* pId = taosArrayGet(execInfo.pTaskList, i);
-    STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, pId, sizeof(*pId));
+    STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
 
     bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
     if (!existed) {
-      taosArrayPush(pRemoveTaskList, pId);
+      taosArrayPush(pRemovedTasks, pId);
     }
   }
 
-  for(int32_t i = 0; i < taosArrayGetSize(pRemoveTaskList); ++i) {
+  for(int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) {
-    STaskId* pId = taosArrayGet(pRemoveTaskList, i);
+    STaskId* pId = taosArrayGet(pRemovedTasks, i);
-    doRemoveFromTask(&execNodeList, pId);
+    doRemoveTasks(&execInfo, pId);
   }
 
-  mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemoveTaskList),
+  mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
-         (int32_t) taosArrayGetSize(execNodeList.pTaskList));
+         (int32_t) taosArrayGetSize(execInfo.pTaskList));
 
   int32_t size = taosArrayGetSize(pNodeSnapshot);
   SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry));
-  for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) {
+  for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
-    SNodeEntry* p = taosArrayGet(execNodeList.pNodeEntryList, i);
+    SNodeEntry* p = taosArrayGet(execInfo.pNodeEntryList, i);
 
     for(int32_t j = 0; j < size; ++j) {
       SNodeEntry* pEntry = taosArrayGet(pNodeSnapshot, j);
@@ -2255,10 +2312,11 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
     }
   }
 
-  execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+  execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
-  execNodeList.pNodeEntryList = pValidNodeEntryList;
+  execInfo.pNodeEntryList = pValidNodeEntryList;
 
-  taosArrayDestroy(pRemoveTaskList);
+  mDebug("remain %d valid node entries", (int32_t) taosArrayGetSize(pValidNodeEntryList));
+  taosArrayDestroy(pRemovedTasks);
   return 0;
 }
 
@@ -2275,43 +2333,55 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
   int64_t ts = taosGetTimestampSec();
 
   SMnode *pMnode = pMsg->info.node;
-  if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) {
+  if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
-    if (execNodeList.pNodeEntryList != NULL) {
+    if (execInfo.pNodeEntryList != NULL) {
-      execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+      execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
     }
-
+    execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
-    execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode);
   }
 
-  if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) {
+  if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
     mDebug("end to do stream task node change checking, no vgroup exists, do nothing");
-    execNodeList.ts = ts;
+    execInfo.ts = ts;
     atomic_store_32(&mndNodeCheckSentinel, 0);
     return 0;
   }
 
-  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode);
+  bool allVnodeReady = true;
+  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVnodeReady);
+  if (!allVnodeReady) {
+    taosArrayDestroy(pNodeSnapshot);
+    atomic_store_32(&mndNodeCheckSentinel, 0);
+    mWarn("not all vnodes are ready, ignore the exec nodeUpdate check");
+    return 0;
+  }
 
-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
   removeExpirednodeEntryAndTask(pNodeSnapshot);
 
-  SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot);
+  SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
   if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) {
+
+    // kill current active checkpoint transaction, since the transaction is vnode wide.
+    doKillActiveCheckpointTrans(pMnode);
     code = mndProcessVgroupChange(pMnode, &changeInfo);
 
     // keep the new vnode snapshot
     if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) {
       mDebug("create trans successfully, update cached node list");
-      taosArrayDestroy(execNodeList.pNodeEntryList);
+      taosArrayDestroy(execInfo.pNodeEntryList);
-      execNodeList.pNodeEntryList = pNodeSnapshot;
+      execInfo.pNodeEntryList = pNodeSnapshot;
-      execNodeList.ts = ts;
+      execInfo.ts = ts;
+    } else {
+      mDebug("unexpect code during create nodeUpdate trans, code:%s", tstrerror(code));
+      taosArrayDestroy(pNodeSnapshot);
     }
   } else {
     mDebug("no update found in nodeList");
     taosArrayDestroy(pNodeSnapshot);
   }
 
-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);
   taosArrayDestroy(changeInfo.pUpdateNodeList);
   taosHashCleanup(changeInfo.pDBMap);
 
@@ -2324,10 +2394,6 @@ typedef struct SMStreamNodeCheckMsg {
   int8_t placeHolder;  // // to fix windows compile error, define place holder
 } SMStreamNodeCheckMsg;
 
-typedef struct SMStreamTaskResetMsg {
-  int8_t placeHolder;
-} SMStreamTaskResetMsg;
-
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
   SMnode *pMnode = pReq->info.node;
   SSdb   *pSdb = pMnode->pSdb;
@@ -2343,7 +2409,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
   return 0;
 }
 
-void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) {
+void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
   int32_t level = taosArrayGetSize(pStream->tasks);
 
   for (int32_t i = 0; i < level; i++) {
@@ -2368,7 +2434,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) {
   }
 }
 
-void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode) {
+void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
   int32_t level = taosArrayGetSize(pStream->tasks);
   for (int32_t i = 0; i < level; i++) {
     SArray *pLevel = taosArrayGetP(pStream->tasks, i);
@@ -2399,7 +2465,7 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode
   ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
 }
 
-static STrans* doCreateTrans(SMnode* pMnode, SStreamObj* pStream, const char* name) {
+STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name) {
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, name);
   if (pTrans == NULL) {
     mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
@@ -2451,7 +2517,7 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
       pReq->streamId = pTask->id.streamId;
 
       STransAction action = {0};
-      initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet);
+      initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet, 0);
       if (mndTransAppendRedoAction(pTrans, &action) != 0) {
         taosMemoryFree(pReq);
         taosWUnLockLatch(&pStream->lock);
@@ -2482,29 +2548,26 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
   return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
-int32_t mndResetFromCheckpoint(SMnode* pMnode) {
+int32_t doKillActiveCheckpointTrans(SMnode *pMnode) {
-  // find the checkpoint trans id
   int32_t transId = 0;
+  SSdb   *pSdb = pMnode->pSdb;
+  STrans *pTrans = NULL;
+  void   *pIter = NULL;
 
-  {
+  while (1) {
-    SSdb   *pSdb = pMnode->pSdb;
+    pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans);
-    STrans *pTrans = NULL;
+    if (pIter == NULL) {
-    void* pIter = NULL;
+      break;
-    while (1) {
-      pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans);
-      if (pIter == NULL) {
-        break;
-      }
-
-      if (strncmp(pTrans->opername, MND_STREAM_CHECKPOINT_NAME, tListLen(pTrans->opername) - 1) == 0) {
-        transId = pTrans->id;
-        sdbRelease(pSdb, pTrans);
-        sdbCancelFetch(pSdb, pIter);
-        break;
-      }
-
-      sdbRelease(pSdb, pTrans);
     }
+
+    if (strncmp(pTrans->opername, MND_STREAM_CHECKPOINT_NAME, tListLen(pTrans->opername) - 1) == 0) {
+      transId = pTrans->id;
+      sdbRelease(pSdb, pTrans);
+      sdbCancelFetch(pSdb, pIter);
+      break;
+    }
+
+    sdbRelease(pSdb, pTrans);
   }
 
   if (transId == 0) {
@@ -2512,8 +2575,16 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
     return TSDB_CODE_SUCCESS;
   }
 
-  STrans* pTrans = mndAcquireTrans(pMnode, transId);
+  pTrans = mndAcquireTrans(pMnode, transId);
+  mInfo("kill checkpoint trans:%d", transId);
+
   mndKillTrans(pMnode, pTrans);
+  mndReleaseTrans(pMnode, pTrans);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mndResetFromCheckpoint(SMnode* pMnode) {
+  doKillActiveCheckpointTrans(pMnode);
 
   // set all tasks status to be normal, refactor later to be stream level, instead of vnode level.
   SSdb *pSdb = pMnode->pSdb;
@@ -2525,6 +2596,7 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
       break;
     }
 
+    // todo this transaction should exist be only one
     mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid);
     int32_t code = createStreamResetStatusTrans(pMnode, pStream);
     if (code != TSDB_CODE_SUCCESS) {
@@ -2536,6 +2608,43 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
   return 0;
 }
 
+int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) {
+  int32_t num = taosArrayGetSize(pNodeList);
+
+  for (int k = 0; k < num; ++k) {
+    int32_t* pVgId = taosArrayGet(pNodeList, k);
+
+    int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
+    for (int i = 0; i < numOfNodes; ++i) {
+      SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
+
+      if (pNodeEntry->nodeId == *pVgId) {
+        mInfo("vgId:%d expired in stream task, needs update nodeEp", *pVgId);
+        pNodeEntry->stageUpdated = true;
+        break;
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) {
+  int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
+  for(int32_t j = 0; j < numOfNodes; ++j) {
+    SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j);
+    if (pNodeEntry->nodeId == pTaskEntry->nodeId) {
+
+      mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId,
+            pTaskEntry->stage, stage, pTaskEntry->id.taskId);
+
+      pNodeEntry->stageUpdated = true;
+      pTaskEntry->stage = stage;
+      break;
+    }
+  }
+}
+
 int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   SMnode      *pMnode = pReq->info.node;
   SStreamHbMsg req = {0};
@@ -2555,35 +2664,26 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
 
   mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks);
 
-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
-  int32_t numOfExisted = taosHashGetSize(execNodeList.pTaskMap);
+  int32_t numOfExisted = taosHashGetSize(execInfo.pTaskMap);
   if (numOfExisted == 0) {
     doExtractTasksFromStream(pMnode);
   }
 
+  setNodeEpsetExpiredFlag(req.pUpdateNodes);
+
   for (int32_t i = 0; i < req.numOfTasks; ++i) {
     STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i);
-    STaskStatusEntry *pEntry = taosHashGet(execNodeList.pTaskMap, &p->id, sizeof(p->id));
+    STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id));
-    if (pEntry == NULL) {
+    if (pTaskEntry == NULL) {
       mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId);
       continue;
     }
 
-    if (p->stage != pEntry->stage && pEntry->stage != -1) {
+    if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) {
-      int32_t numOfNodes = taosArrayGetSize(execNodeList.pNodeEntryList);
+      updateStageInfo(pTaskEntry, p->stage);
-      for(int32_t j = 0; j < numOfNodes; ++j) {
-        SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, j);
-        if (pNodeEntry->nodeId == pEntry->nodeId) {
-          mInfo("vgId:%d stage updated, from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64,
-                pEntry->nodeId, pEntry->stage, p->stage, pEntry->id.taskId);
-
-          pNodeEntry->stageUpdated = true;
-          pEntry->stage = p->stage;
-          break;
-        }
-      }
     } else {
-      streamTaskStatusCopy(pEntry, p);
+      streamTaskStatusCopy(pTaskEntry, p);
       if (p->activeCheckpointId != 0) {
         if (activeCheckpointId != 0) {
           ASSERT(activeCheckpointId == p->activeCheckpointId);
@@ -2597,26 +2697,32 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
       }
     }
 
-    pEntry->status = p->status;
+    pTaskEntry->status = p->status;
-    if (p->status != TASK_STATUS__NORMAL) {
+    if (p->status != TASK_STATUS__READY) {
-      mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamGetTaskStatusStr(p->status));
+      mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status));
     }
   }
 
   // current checkpoint is failed, rollback from the checkpoint trans
   // kill the checkpoint trans and then set all tasks status to be normal
   if (checkpointFailed && activeCheckpointId != 0) {
-    if (execNodeList.activeCheckpoint != activeCheckpointId) {
+    bool allReady = true;
-      mInfo("checkpointId:%"PRId64" failed, issue task-reset trans to reset all tasks status", activeCheckpointId);
+    SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady);
-      execNodeList.activeCheckpoint = activeCheckpointId;
+    taosArrayDestroy(p);
+
+    if (allReady) {
+      // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
+      mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status",
+            execInfo.activeCheckpoint);
       mndResetFromCheckpoint(pMnode);
     } else {
-      mDebug("checkpoint:%"PRId64" reset has issued already, ignore it", activeCheckpointId);
+      mInfo("not all vgroups are ready, wait for next HB from stream tasks");
     }
   }
 
-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);
 
   taosArrayDestroy(req.pTaskStatus);
+  taosArrayDestroy(req.pUpdateNodes);
   return TSDB_CODE_SUCCESS;
 }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -754,10 +754,14 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup
   for (int32_t v = 0; v < pVgroup->replica; ++v) {
     SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
     SDnodeObj *pDnode = taosArrayGet(pArray, v);
-    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+    if (pDnode == NULL) {
       terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
       return -1;
     }
+    if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
+      return -1;
+    }
 
     int64_t vgMem = mndGetVgroupMemory(pMnode, pDb, pVgroup);
     if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@@ -1180,10 +1184,14 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro
     }
     if (used) continue;
 
-    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+    if (pDnode == NULL) {
       terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
       return -1;
     }
+    if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
+      return -1;
+    }
 
     int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
     if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@@ -1912,7 +1920,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
     if (numOfVnodes >= pNew1->numOfSupportVnodes) {
       mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes,
              pNew1->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
       goto _OVER;
     }
 
@@ -1935,7 +1943,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
     if (numOfVnodes >= pNew2->numOfSupportVnodes) {
       mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes,
              pNew2->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
       goto _OVER;
     }
     int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
@@ -1956,7 +1964,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
     if (numOfVnodes >= pNew3->numOfSupportVnodes) {
       mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes,
              pNew3->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
       goto _OVER;
     }
     int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);

source/dnode/snode/src/snode.c

@@ -92,11 +92,13 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
     }
   }
 
-  qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64
+  char* p = NULL;
-         " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms",
+  streamTaskGetStatus(pTask, &p);
+
+  qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
+        " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms",
         SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-         pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
+        pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.triggerParam);
-         pTask->info.fillHistory, pTask->info.triggerParam);
 
   return 0;
 }
@@ -174,9 +176,15 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
 
   int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta);
   taosWUnLockLatch(&pSnode->pMeta->lock);
-  qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr,
-         streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks);
 
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
+  qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE,
+         pTask->id.idStr, p, numOfTasks);
+
+  EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+  streamTaskHandleEvent(pTask->status.pSM, event);
   streamTaskCheckDownstream(pTask);
   return 0;
 }
@@ -352,10 +360,10 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) {
   if (pTask != NULL) {
     rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
     streamMetaReleaseTask(pSnode->pMeta, pTask);
-
+    char* p = NULL;
-    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+    streamTaskGetStatus(pTask, &p);
     qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d",
-            pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+            pTask->id.idStr, p, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
   } else {
     rsp.status = TASK_DOWNSTREAM_NOT_READY;
     qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d",

source/dnode/vnode/src/inc/tq.h

@@ -126,7 +126,6 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
 int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
 int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
                       int32_t type, int32_t vgId);
-//int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId);
 int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
 
 // tqMeta
@@ -134,7 +133,6 @@ int32_t tqMetaOpen(STQ* pTq);
 int32_t tqMetaClose(STQ* pTq);
 int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
 int32_t tqMetaDeleteHandle(STQ* pTq, const char* key);
-//int32_t tqMetaRestoreHandle(STQ* pTq);
 int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen);
 int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
 int32_t tqMetaRestoreCheckInfo(STQ* pTq);
@@ -160,8 +158,8 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
 // tqStream
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
 int32_t tqScanWal(STQ* pTq);
-int32_t tqCheckAndRunStreamTask(STQ* pTq);
+int32_t tqStartStreamTask(STQ* pTq);
-int32_t tqStartStreamTasks(STQ* pTq);
+int32_t tqResetStreamTaskStatus(STQ* pTq);
 int32_t tqStopStreamTasks(STQ* pTq);
 
 // tq util
@@ -171,6 +169,12 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
                         int32_t type, int64_t sver, int64_t ever);
 int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset);
 void    tqUpdateNodeStage(STQ* pTq, bool isLeader);
+int32_t setDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
+                               SSubmitTbData* pTableData, const char* id);
+int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
+
+SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
+                                       SSDataBlock* pDataBlock, SArray* pTagArray);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -231,7 +231,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
-int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq);
+int32_t tqLaunchStreamTaskAsync(STQ* pTq);
 
 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -35,8 +35,8 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) {
   return code;
 }
 
-int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) {
+int32_t tdProcessTSmaCreate(SSma *pSma, int64_t ver, const char *msg) {
-  int32_t code = tdProcessTSmaCreateImpl(pSma, version, msg);
+  int32_t code = tdProcessTSmaCreateImpl(pSma, ver, msg);
 
   return code;
 }
@@ -109,7 +109,7 @@ _exit:
  * @param pMsg
  * @return int32_t
  */
-static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) {
+static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t ver, const char *pMsg) {
   int32_t        code = 0;
   int32_t        lino = 0;
   SSmaCfg       *pCfg = (SSmaCfg *)pMsg;
@@ -118,7 +118,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *
 
   if (TD_VID(pSma->pVnode) == pCfg->dstVgId) {
     // create tsma meta in dstVgId
-    if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) {
+    if (metaCreateTSma(SMA_META(pSma), ver, pCfg) < 0) {
       code = terrno;
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -130,7 +130,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *
     pReq.schemaRow = pCfg->schemaRow;
     pReq.schemaTag = pCfg->schemaTag;
 
-    if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) {
+    if (metaCreateSTable(SMA_META(pSma), ver, &pReq) < 0) {
       code = terrno;
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -154,94 +154,36 @@ _exit:
   return code;
 }
 
-int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema,
+int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, int64_t suid,
-                         SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName,
+                         const char *stbFullName, SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
-                         SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
   int32_t      code = 0;
   int32_t      lino = 0;
   void        *pBuf = NULL;
   int32_t      len = 0;
   SSubmitReq2 *pReq = NULL;
   SArray      *tagArray = NULL;
-  SArray      *createTbArray = NULL;
-  SArray      *pVals = NULL;
 
-  int32_t sz = taosArrayGetSize(pBlocks);
+  int32_t numOfBlocks = taosArrayGetSize(pBlocks);
 
   tagArray = taosArrayInit(1, sizeof(STagVal));
-  createTbArray = taosArrayInit(sz, POINTER_BYTES);
   pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2));
-  pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));
 
-  if(!tagArray || !createTbArray || !pReq || !pReq->aSubmitTbData) {
+  if (!tagArray || !pReq) {
     code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  // create table req
+  pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));
-  if (createTb) {
+  if (pReq->aSubmitTbData == NULL) {
-    for (int32_t i = 0; i < sz; ++i) {
+    code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
-      SSDataBlock   *pDataBlock = taosArrayGet(pBlocks, i);
+    TSDB_CHECK_CODE(code, lino, _exit);
-      SVCreateTbReq *pCreateTbReq = NULL;
-      if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
-        taosArrayPush(createTbArray, &pCreateTbReq);
-        continue;
-      }
-
-      if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        TSDB_CHECK_CODE(code, lino, _exit);
-      };
-
-      // don't move to the end of loop as to destroy in the end of func when error occur
-      taosArrayPush(createTbArray, &pCreateTbReq);
-
-      // set const
-      pCreateTbReq->flags = 0;
-      pCreateTbReq->type = TSDB_CHILD_TABLE;
-      pCreateTbReq->ctb.suid = suid;
-
-      // set super table name
-      SName name = {0};
-      tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-      pCreateTbReq->ctb.stbName = taosStrdup((char *)tNameGetTableName(&name));  // taosStrdup(stbFullName);
-
-      // set tag content
-      taosArrayClear(tagArray);
-      STagVal tagVal = {
-          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
-          .type = TSDB_DATA_TYPE_UBIGINT,
-          .i64 = (int64_t)pDataBlock->info.id.groupId,
-      };
-      taosArrayPush(tagArray, &tagVal);
-      pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
-
-      STag *pTag = NULL;
-      tTagNew(tagArray, 1, false, &pTag);
-      if (pTag == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        TSDB_CHECK_CODE(code, lino, _exit);
-      }
-      pCreateTbReq->ctb.pTag = (uint8_t *)pTag;
-
-      // set tag name
-      SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-      char    tagNameStr[TSDB_COL_NAME_LEN] = {0};
-      strcpy(tagNameStr, "group_id");
-      taosArrayPush(tagName, tagNameStr);
-      pCreateTbReq->ctb.tagName = tagName;
-
-      // set table name
-      if (pDataBlock->info.parTbName[0]) {
-        pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
-      } else {
-        pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId);
-      }
-    }
   }
 
+  SHashObj *pTableIndexMap =
+      taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+
   // SSubmitTbData req
-  for (int32_t i = 0; i < sz; ++i) {
+  for (int32_t i = 0; i < numOfBlocks; ++i) {
     SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i);
     if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
       pDeleteReq->suid = suid;
@@ -250,62 +192,42 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
       continue;
     }
 
-    int32_t rows = pDataBlock->info.rows;
+    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE,};
 
-    SSubmitTbData tbData = {0};
+    int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
+    tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray);
 
-    if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow *)))) {
+    {
-      code = terrno;
+      uint64_t groupId = pDataBlock->info.id.groupId;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-    tbData.suid = suid;
-    tbData.uid = 0;  // uid is assigned by vnode
-    tbData.sver = pTSchema->version;
 
-    if (createTb) {
+      int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
-      tbData.pCreateTbReq = taosArrayGetP(createTbArray, i);
+      if (index == NULL) {  // no data yet, append it
-      if (tbData.pCreateTbReq) tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
-    }
+        if (code != TSDB_CODE_SUCCESS) {
+          continue;
+        }
 
-    if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) {
+        taosArrayPush(pReq->aSubmitTbData, &tbData);
-      taosArrayDestroy(tbData.aRowP);
-      code = terrno;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
 
-    for (int32_t j = 0; j < rows; ++j) {
+        int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1;
-      taosArrayClear(pVals);
+        taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
-      for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
+      } else {
-        const STColumn  *pCol = &pTSchema->columns[k];
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
-        SColumnInfoData *pColData = taosArrayGet(pDataBlock->pDataBlock, k);
+        if (code != TSDB_CODE_SUCCESS) {
-        if (colDataIsNull_s(pColData, j)) {
+          continue;
-          SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
+        }
-          taosArrayPush(pVals, &cv);
+
-        } else {
+        SSubmitTbData *pExisted = taosArrayGet(pReq->aSubmitTbData, *index);
-          void *data = colDataGetData(pColData, j);
+        code = doMergeExistedRows(pExisted, &tbData, "id");
-          if (IS_STR_DATA_TYPE(pCol->type)) {
+        if (code != TSDB_CODE_SUCCESS) {
-            SValue  sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)};  // address copy, no value
+          continue;
-            SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
-            taosArrayPush(pVals, &cv);
-          } else {
-            SValue sv;
-            memcpy(&sv.val, data, tDataTypes[pCol->type].bytes);
-            SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
-            taosArrayPush(pVals, &cv);
-          }
         }
       }
-      SRow *pRow = NULL;
-      if ((code = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) {
-        tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
-        TSDB_CHECK_CODE(code, lino, _exit);
-      }
-      taosArrayPush(tbData.aRowP, &pRow);
     }
-
-    taosArrayPush(pReq->aSubmitTbData, &tbData);
   }
 
+  taosHashCleanup(pTableIndexMap);
+
   // encode
   tEncodeSize(tEncodeSubmitReq, pReq, len, code);
   if (TSDB_CODE_SUCCESS == code) {
@@ -327,11 +249,10 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
     }
     tEncoderClear(&encoder);
   }
+
 _exit:
-  taosArrayDestroy(createTbArray);
   taosArrayDestroy(tagArray);
-  taosArrayDestroy(pVals);
+  if (pReq != NULL) {
-  if (pReq) {
     tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
     taosMemoryFree(pReq);
   }
@@ -442,8 +363,8 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char
   void           *pSubmitReq = NULL;
   int32_t         contLen = 0;
 
-  code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true,
+  code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, pTsmaStat->pTSma->dstTbUid,
-                          pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen);
+                          pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) {

source/dnode/vnode/src/tq/tq.c

@@ -20,6 +20,12 @@ typedef struct {
   int8_t inited;
 } STqMgmt;
 
+typedef struct STaskUpdateEntry {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t transId;
+} STaskUpdateEntry;
+
 static STqMgmt tqMgmt = {0};
 
 // 0: not init
@@ -820,28 +826,29 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   }
 
   // sink
-  if (pTask->outputInfo.type == TASK_OUTPUT__SMA) {
+  STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
-    pTask->outputInfo.smaSink.vnode = pTq->pVnode;
+  if (pOutputInfo->type == TASK_OUTPUT__SMA) {
-    pTask->outputInfo.smaSink.smaSink = smaHandleRes;
+    pOutputInfo->smaSink.vnode = pTq->pVnode;
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
+    pOutputInfo->smaSink.smaSink = smaHandleRes;
-    pTask->outputInfo.tbSink.vnode = pTq->pVnode;
+  } else if (pOutputInfo->type == TASK_OUTPUT__TABLE) {
-    pTask->outputInfo.tbSink.tbSinkFunc = tqSinkDataIntoDstTable;
+    pOutputInfo->tbSink.vnode = pTq->pVnode;
+    pOutputInfo->tbSink.tbSinkFunc = tqSinkDataIntoDstTable;
 
     int32_t   ver1 = 1;
     SMetaInfo info = {0};
-    code = metaGetInfo(pTq->pVnode->pMeta, pTask->outputInfo.tbSink.stbUid, &info, NULL);
+    code = metaGetInfo(pTq->pVnode->pMeta, pOutputInfo->tbSink.stbUid, &info, NULL);
     if (code == TSDB_CODE_SUCCESS) {
       ver1 = info.skmVer;
     }
 
-    SSchemaWrapper* pschemaWrapper = pTask->outputInfo.tbSink.pSchemaWrapper;
+    SSchemaWrapper* pschemaWrapper = pOutputInfo->tbSink.pSchemaWrapper;
-    pTask->outputInfo.tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
+    pOutputInfo->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
-    if (pTask->outputInfo.tbSink.pTSchema == NULL) {
+    if (pOutputInfo->tbSink.pTSchema == NULL) {
       return -1;
     }
 
-    pTask->outputInfo.tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
+    pOutputInfo->tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
-    tSimpleHashSetFreeFp(pTask->outputInfo.tbSink.pTblInfo, freePtr);
+    tSimpleHashSetFreeFp(pOutputInfo->tbSink.pTblInfo, freePtr);
   }
 
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@@ -849,11 +856,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
     pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId);
   }
 
-  // reset the task status from unfinished transaction
+//  // reset the task status from unfinished transaction
-  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+//  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-    tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
+//    tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
-    pTask->status.taskStatus = TASK_STATUS__NORMAL;
+//    pTask->status.taskStatus = TASK_STATUS__READY;
-  }
+//  }
 
   streamTaskResetUpstreamStageInfo(pTask);
   streamSetupScheduleTrigger(pTask);
@@ -866,20 +873,23 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
            pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
   }
 
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
   if (pTask->info.fillHistory) {
     tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
            " nextProcessVer:%" PRId64
            " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
            vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-           pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
+           pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
-           pTask->info.fillHistory, (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
+           (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
   } else {
     tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
            " nextProcessVer:%" PRId64
            " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
            vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-           pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
+           pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
-           pTask->info.fillHistory, (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
+           (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
   }
 
   return 0;
@@ -921,9 +931,10 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
       rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
       streamMetaReleaseTask(pMeta, pTask);
 
-      const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+      char* p = NULL;
+      streamTaskGetStatus(pTask, &p);
       tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d",
-              pTask->id.idStr, pStatus, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+              pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
     } else {
       rsp.status = TASK_DOWNSTREAM_NOT_READY;
       tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64
@@ -957,6 +968,12 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) {
   tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d",
           rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
 
+  if (!vnodeIsRoleLeader(pTq->pVnode)) {
+    tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId,
+            rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId);
+    return code;
+  }
+
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.streamId, rsp.upstreamTaskId);
   if (pTask == NULL) {
     tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
@@ -1026,11 +1043,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
       SStreamTask* p = streamMetaAcquireTask(pStreamMeta, streamId, taskId);
 
       bool restored = pTq->pVnode->restored;
-      if (p != NULL && restored) {
+      if (p != NULL && restored && p->info.fillHistory == 0) {
-        p->execInfo.init = taosGetTimestampMs();
+        EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
-        tqDebug("s-task:%s set the init ts:%"PRId64, p->id.idStr, p->execInfo.init);
+        streamTaskHandleEvent(p->status.pSM, event);
-
-        streamTaskCheckDownstream(p);
       } else if (!restored) {
         tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId);
       }
@@ -1064,7 +1079,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
 
   // do recovery step1
   const char* id = pTask->id.idStr;
-  const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+  char*       pStatus = NULL;
+  streamTaskGetStatus(pTask, &pStatus);
 
   // avoid multi-thread exec
   while(1) {
@@ -1117,8 +1133,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
   streamScanHistoryData(pTask);
 
   double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
-  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE) {
-    int8_t status = streamTaskSetSchedStatusInActive(pTask);
+    int8_t status = streamTaskSetSchedStatusInactive(pTask);
     tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status);
 
     atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
@@ -1127,12 +1143,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   // the following procedure should be executed, no matter status is stop/pause or not
-  tqDebug("s-task:%s scan-history stage(step 1) ended, elapsed time:%.2fs", id, el);
+  tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, el);
 
   if (pTask->info.fillHistory) {
     SVersionRange* pRange = NULL;
     SStreamTask*   pStreamTask = NULL;
-    bool           done = false;
 
     // 1. get the related stream task
     pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
@@ -1151,10 +1166,10 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
     }
 
     ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE);
-
+#if 0
     // 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the
     // stream task get ready for scan history data
-    while (pStreamTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+    while (streamTaskGetStatus(pStreamTask, NULL) == TASK_STATUS__SCAN_HISTORY) {
       tqDebug(
           "s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms",
           id, pTask->info.taskLevel, pStreamTask->id.idStr, streamGetTaskStatusStr(pStreamTask->status.taskStatus));
@@ -1199,6 +1214,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
       pStreamTask->status.keepTaskStatus = status;
       pStreamTask->status.taskStatus = TASK_STATUS__HALT;
 
+      // wal scan not start yet, reset it to be the start position
       nextProcessedVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader);
       if (nextProcessedVer == -1) {
         nextProcessedVer = pStreamTask->dataRange.range.maxVer + 1;
@@ -1211,21 +1227,26 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
       taosThreadMutexUnlock(&pStreamTask->lock);
       break;
     }
+#endif
+
+    streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
+    int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer;
 
     // if it's an source task, extract the last version in wal.
     pRange = &pTask->dataRange.range;
-    done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
+    bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
     pTask->execInfo.step2Start = taosGetTimestampMs();
 
     if (done) {
       qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, 0.0);
       streamTaskPutTranstateIntoInputQ(pTask);
+//      streamTaskRestoreStatus(pTask);
 
-      if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+//      if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-        pTask->status.keepTaskStatus = TASK_STATUS__NORMAL;
+//        pTask->status.keepTaskStatus = TASK_STATUS__READY;
-        qDebug("s-task:%s prev status is %s, update the kept status to be:%s when after step 2", id,
+//        qDebug("s-task:%s prev status is %s, update the kept status to be:%s when after step 2", id,
-               streamGetTaskStatusStr(TASK_STATUS__PAUSE), streamGetTaskStatusStr(pTask->status.keepTaskStatus));
+//               streamGetTaskStatusStr(TASK_STATUS__PAUSE), streamGetTaskStatusStr(pTask->status.keepTaskStatus));
-      }
+//      }
 
       streamExecTask(pTask);  // exec directly
     } else {
@@ -1245,35 +1266,24 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
       tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer,
               pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE);
 
-      /*int8_t status = */streamTaskSetSchedStatusInActive(pTask);
+      /*int8_t status = */streamTaskSetSchedStatusInactive(pTask);
-
-      // the fill-history task starts to process data in wal, let's set it status to be normal now
-      if (pTask->info.fillHistory == 1 && !streamTaskShouldStop(&pTask->status)) {
-        streamSetStatusNormal(pTask);
-      }
 
+      // now the fill-history task starts to scan data from wal files.
+      streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
       tqScanWalAsync(pTq, false);
     }
     streamMetaReleaseTask(pMeta, pStreamTask);
 
   } else {
     STimeWindow* pWindow = &pTask->dataRange.window;
+    ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));
 
-    if (pTask->hTaskInfo.id.taskId == 0) {
+    // Not update the fill-history time window until the state transfer is completed if the related fill-history task
-      *pWindow = (STimeWindow){INT64_MIN, INT64_MAX};
+    // exists.
-      tqDebug(
+    tqDebug(
-          "s-task:%s scan-history in stream time window completed, no related fill-history task, reset the time "
+        "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, startVer:%" PRId64
-          "window:%" PRId64 " - %" PRId64,
+        ", window:%" PRId64 " - %" PRId64,
-          id, pWindow->skey, pWindow->ekey);
+        id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey);
-      qStreamInfoResetTimewindowFilter(pTask->exec.pExecutor);
-    } else {
-      // when related fill-history task exists, update the fill-history time window only when the
-      // state transfer is completed.
-      tqDebug(
-          "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, start "
-          "ver:%" PRId64 ", window:%" PRId64 " - %" PRId64,
-          id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey);
-    }
 
     code = streamTaskScanHistoryDataComplete(pTask);
   }
@@ -1352,7 +1362,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   int32_t vgId = TD_VID(pTq->pVnode);
 
   if (taskId == STREAM_EXEC_TASK_STATUS_CHECK_ID) {
-    tqCheckAndRunStreamTask(pTq);
+    tqStartStreamTask(pTq);
     return 0;
   }
 
@@ -1362,17 +1372,16 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId);
-  if (pTask != NULL) {
+  if (pTask != NULL) { // even in halt status, the data in inputQ must be processed
-    // even in halt status, the data in inputQ must be processed
+    char* p = NULL;
-    int8_t st = pTask->status.taskStatus;
+    if (streamTaskReadyToRun(pTask, &p)) {
-    if (st == TASK_STATUS__NORMAL || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK) {
       tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr,
               pTask->chkInfo.nextProcessVer);
       streamExecTask(pTask);
     } else {
-      int8_t status = streamTaskSetSchedStatusInActive(pTask);
+      int8_t status = streamTaskSetSchedStatusInactive(pTask);
       tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
-              pTask->id.idStr, streamGetTaskStatusStr(st), status);
+              pTask->id.idStr, p, status);
     }
 
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@@ -1446,7 +1455,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
   if (pTask != NULL) {
     // drop the related fill-history task firstly
-    if (pTask->hTaskInfo.id.taskId != 0) {
+    if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
       STaskId* pHTaskId = &pTask->hTaskInfo.id;
       streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId);
       tqDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId);
@@ -1486,7 +1495,7 @@ int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
   streamTaskPause(pTask, pMeta);
 
   SStreamTask* pHistoryTask = NULL;
-  if (pTask->hTaskInfo.id.taskId != 0) {
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId);
     if (pHistoryTask == NULL) {
       tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
@@ -1514,17 +1523,19 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
     return -1;
   }
 
-  // todo: handle the case: resume from halt to pause/ from halt to normal/ from pause to normal
+  streamTaskResume(pTask);
-  streamTaskResume(pTask, pTq->pStreamMeta);
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
 
   int32_t level = pTask->info.taskLevel;
   if (level == TASK_LEVEL__SINK) {
+    if (status == TASK_STATUS__UNINIT) {
+
+    }
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     return 0;
   }
 
-  int8_t  status = pTask->status.taskStatus;
+  if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
-  if (status == TASK_STATUS__NORMAL || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
     // no lock needs to secure the access of the version
     if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) {
       // discard all the data  when the stream task is suspended.
@@ -1537,14 +1548,18 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
               vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus);
     }
 
-    if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory &&
+    if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
-        pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
       streamStartScanHistoryAsync(pTask, igUntreated);
-    } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputInfo.queue) == 0)) {
+    } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
       tqScanWalAsync(pTq, false);
     } else {
       streamSchedExec(pTask);
     }
+  } else if (status == TASK_STATUS__UNINIT) {
+    if (pTask->info.fillHistory == 0) {
+      EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+      streamTaskHandleEvent(pTask->status.pSM, event);
+    }
   }
 
   streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@@ -1683,7 +1698,6 @@ FAIL:
   return -1;
 }
 
-// todo error code cannot be return, since this is invoked by an mnode-launched transaction.
 int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) {
   int32_t      vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
@@ -1694,7 +1708,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   // disable auto rsp to mnode
   pRsp->info.handle = NULL;
 
-  // todo: add counter to make sure other tasks would not be trapped in checkpoint state
   SStreamCheckpointSourceReq req = {0};
   if (!vnodeIsRoleLeader(pTq->pVnode)) {
     tqDebug("vgId:%d not leader, ignore checkpoint-source msg", vgId);
@@ -1725,6 +1738,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   }
   tDecoderClear(&decoder);
 
+  // todo handle failure to reset from checkpoint procedure
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
   if (pTask == NULL) {
     tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. it may have been destroyed already", vgId,
@@ -1735,6 +1749,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
     return TSDB_CODE_SUCCESS;
   }
 
+  // todo handle failure to reset from checkpoint procedure
   // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req.
   if (pTask->status.downstreamReady != 1) {
     pTask->chkInfo.failedId = req.checkpointId;   // record the latest failed checkpoint id
@@ -1750,8 +1765,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
     return TSDB_CODE_SUCCESS;
   }
 
+  // todo save the checkpoint failed info
   taosThreadMutexLock(&pTask->lock);
-  if (pTask->status.taskStatus == TASK_STATUS__HALT) {
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
+
+  if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {
     qError("s-task:%s not ready for checkpoint, since it is halt, ignore this checkpoint:%" PRId64 ", set it failure",
            pTask->id.idStr, req.checkpointId);
     taosThreadMutexUnlock(&pTask->lock);
@@ -1832,7 +1850,6 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
   char*        msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   int32_t      len = pMsg->contLen - sizeof(SMsgHead);
   SRpcMsg      rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
-  bool         allStopped = false;
 
   SStreamTaskNodeUpdateMsg req = {0};
 
@@ -1863,17 +1880,43 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   SStreamTask* pTask = *ppTask;
-  tqDebug("s-task:%s receive nodeEp update msg from mnode", pTask->id.idStr);
 
+  if (pMeta->updateInfo.transId != req.transId) {
+    pMeta->updateInfo.transId = req.transId;
+    tqDebug("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId);
+    // info needs to be kept till the new trans to update the nodeEp arrived.
+    taosHashClear(pMeta->updateInfo.pTasks);
+  } else {
+    tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId);
+  }
+
+  STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId};
+  void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
+  if (exist != NULL) {
+    tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId,
+            req.transId);
+    rsp.code = TSDB_CODE_SUCCESS;
+    taosWUnLockLatch(&pMeta->lock);
+    taosArrayDestroy(req.pNodeList);
+    return rsp.code;
+  }
+
+  taosWUnLockLatch(&pMeta->lock);
+
+  // the following two functions should not be executed within the scope of meta lock to avoid deadlock
   streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
-  streamSetStatusNormal(pTask);
+  streamTaskResetStatus(pTask);
+
+  // continue after lock the meta again
+  taosWLockLatch(&pMeta->lock);
 
   SStreamTask** ppHTask = NULL;
-  if (pTask->hTaskInfo.id.taskId != 0) {
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id));
     if (ppHTask == NULL || *ppHTask == NULL) {
       tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already",
               pMeta->vgId, req.taskId);
+      CLEAR_RELATED_FILLHISTORY_TASK(pTask);
     } else {
       tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr);
       streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList);
@@ -1892,12 +1935,14 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   streamTaskStop(pTask);
-  taosHashPut(pMeta->pUpdateTaskSet, &pTask->id, sizeof(pTask->id), NULL, 0);
+
+  // keep the already handled info
+  taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0);
 
   if (ppHTask != NULL) {
     streamTaskStop(*ppHTask);
     tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr);
-    taosHashPut(pMeta->pUpdateTaskSet, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0);
+    taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0);
   } else {
     tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr);
   }
@@ -1906,25 +1951,30 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
 
   // possibly only handle the stream task.
   int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
-  int32_t updateTasks = taosHashGetSize(pMeta->pUpdateTaskSet);
+  int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks);
 
-  pMeta->startInfo.startedAfterNodeUpdate = 1;
+  pMeta->startInfo.startAllTasksFlag = 1;
 
   if (updateTasks < numOfTasks) {
     tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId,
             updateTasks, (numOfTasks - updateTasks));
     taosWUnLockLatch(&pMeta->lock);
   } else {
-    taosHashClear(pMeta->pUpdateTaskSet);
-
     if (!pTq->pVnode->restored) {
       tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId);
-      pMeta->startInfo.startedAfterNodeUpdate = 0;
+      pMeta->startInfo.startAllTasksFlag = 0;
       taosWUnLockLatch(&pMeta->lock);
     } else {
       tqDebug("vgId:%d tasks are all updated and stopped, restart them", vgId);
-
       terrno = 0;
+      taosWUnLockLatch(&pMeta->lock);
+
+      while (streamMetaTaskInTimer(pMeta)) {
+        qDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
+        taosMsleep(100);
+      }
+
+      taosWLockLatch(&pMeta->lock);
       int32_t code = streamMetaReopen(pMeta);
       if (code != 0) {
         tqError("vgId:%d failed to reopen stream meta", vgId);
@@ -1942,8 +1992,8 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
 
       if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) {
         vInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
-        tqStartStreamTasks(pTq);
+        tqResetStreamTaskStatus(pTq);
-        tqCheckAndRunStreamTaskAsync(pTq);
+        tqLaunchStreamTaskAsync(pTq);
       } else {
         vInfo("vgId:%d, follower node not start stream tasks", vgId);
       }
@@ -1970,10 +2020,10 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
   tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
 
   // clear flag set during do checkpoint, and open inputQ for all upstream tasks
-  if (pTask->status.taskStatus == TASK_STATUS__CK) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) {
     streamTaskClearCheckInfo(pTask);
     taosArrayClear(pTask->pReadyMsgList);
-    streamSetStatusNormal(pTask);
+    streamTaskSetStatusReady(pTask);
   }
 
   streamMetaReleaseTask(pMeta, pTask);

source/dnode/vnode/src/tq/tqPush.c

@@ -39,7 +39,7 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) {
   int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta);
   taosRUnLockLatch(&pTq->pStreamMeta->lock);
 
-  tqDebug("handle submit, restore:%d, numOfTasks:%d", pTq->pVnode->restored, numOfTasks);
+//  tqTrace("vgId:%d handle submit, restore:%d, numOfTasks:%d", TD_VID(pTq->pVnode), pTq->pVnode->restored, numOfTasks);
 
   // push data for stream processing:
   // 1. the vnode has already been restored.

source/dnode/vnode/src/tq/tqSink.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <common/tmsg.h>
 #include "tcommon.h"
 #include "tmsg.h"
 #include "tq.h"
@@ -28,19 +29,19 @@ static bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks);
 static int32_t tsAscendingSortFn(const void* p1, const void* p2);
 static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName,
                                   SSubmitTbData* pTableData);
-static int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock,
-                                      SSubmitTbData* pTableData);
 static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
                                        int64_t suid);
 static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks);
 static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen);
-static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id);
+static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id);
 static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo,
                                         const char* dstTableName, int64_t* uid);
 static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id);
-static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
 static bool    isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
-static SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock);
+static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags);
+static SArray* createDefaultTagColName();
+static void    setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
+                                   int64_t gid);
 
 int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
                          const char* pIdStr) {
@@ -138,61 +139,68 @@ static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
+  pCreateTableReq->flags = 0;
+  pCreateTableReq->type = TSDB_CHILD_TABLE;
+  pCreateTableReq->ctb.suid = suid;
+
+  // set super table name
+  SName name = {0};
+  tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+  pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
+
+  pCreateTableReq->ctb.tagNum = numOfTags;
+  return TSDB_CODE_SUCCESS;
+}
+
+SArray* createDefaultTagColName() {
+  SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
+  char    tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
+  taosArrayPush(pTagColNameList, tagNameStr);
+  return pTagColNameList;
+}
+
+void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
+                                int64_t gid) {
+  if (pDataBlock->info.parTbName[0]) {
+    pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
+  } else {
+    pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid);
+  }
+}
+
 static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
                                             int64_t suid) {
   tqDebug("s-task:%s build create table msg", pTask->id.idStr);
 
   STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
   int32_t   rows = pDataBlock->info.rows;
-  SArray*   tagArray = NULL;
+  SArray*   tagArray = taosArrayInit(4, sizeof(STagVal));;
   int32_t   code = 0;
 
   SVCreateTbBatchReq reqs = {0};
-
   SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
   if (NULL == reqs.pArray) {
+    tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno));
     goto _end;
   }
 
   for (int32_t rowId = 0; rowId < rows; rowId++) {
     SVCreateTbReq* pCreateTbReq = &((SVCreateTbReq){0});
 
-    // set const
-    pCreateTbReq->flags = 0;
-    pCreateTbReq->type = TSDB_CHILD_TABLE;
-    pCreateTbReq->ctb.suid = suid;
-
-    // set super table name
-    SName name = {0};
-    tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-    pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));  // taosStrdup(stbFullName);
-
-    // set tag content
     int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
-    if (size == 2) {
+    int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
-      tagArray = taosArrayInit(1, sizeof(STagVal));
-      if (!tagArray) {
-        tdDestroySVCreateTbReq(pCreateTbReq);
-        goto _end;
-      }
 
+    initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
+    taosArrayClear(tagArray);
+
+    if (size == 2) {
       STagVal tagVal = {
           .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
 
       taosArrayPush(tagArray, &tagVal);
-
+      pCreateTbReq->ctb.tagName = createDefaultTagColName();
-      // set tag name
-      SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-      char    tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
-      taosArrayPush(tagName, tagNameStr);
-      pCreateTbReq->ctb.tagName = tagName;
     } else {
-      tagArray = taosArrayInit(size - 1, sizeof(STagVal));
-      if (!tagArray) {
-        tdDestroySVCreateTbReq(pCreateTbReq);
-        goto _end;
-      }
-
       for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
         SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
 
@@ -209,29 +217,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
         taosArrayPush(tagArray, &tagVal);
       }
     }
-    pCreateTbReq->ctb.tagNum = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
 
-    STag* pTag = NULL;
+    tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
-    tTagNew(tagArray, 1, false, &pTag);
     tagArray = taosArrayDestroy(tagArray);
-    if (pTag == NULL) {
+    if (pCreateTbReq->ctb.pTag == NULL) {
       tdDestroySVCreateTbReq(pCreateTbReq);
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _end;
     }
 
-    pCreateTbReq->ctb.pTag = (uint8_t*)pTag;
+    uint64_t gid = pDataBlock->info.id.groupId;
-
+    if (taosArrayGetSize(pDataBlock->pDataBlock) > UD_GROUPID_COLUMN_INDEX) {
-    // set table name
-    if (!pDataBlock->info.parTbName[0]) {
       SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX);
 
+      // todo remove this
       void* pGpIdData = colDataGetData(pGpIdColInfo, rowId);
-      pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData);
+      ASSERT(gid == *(int64_t*)pGpIdData);
-    } else {
-      pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
     }
 
+    setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid);
+
     taosArrayPush(reqs.pArray, pCreateTbReq);
     tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
   }
@@ -330,6 +335,9 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
 
   tqTrace("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id,
           (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (pNew->pCreateTbReq != NULL));
+
+  tdDestroySVCreateTbReq(pNew->pCreateTbReq);
+  taosMemoryFree(pNew->pCreateTbReq);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -390,60 +398,33 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
   return true;
 }
 
-SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock) {
+SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
-  char* ctbName = pDataBlock->info.parTbName;
+                                       SSDataBlock* pDataBlock, SArray* pTagArray) {
-
+  SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
-  SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq));
   if (pCreateTbReq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  // set tag content
+  taosArrayClear(pTagArray);
-  SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
+  initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
-  if (tagArray == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tdDestroySVCreateTbReq(pCreateTbReq);
-    taosMemoryFreeClear(pCreateTbReq);
-    return NULL;
-  }
-
-  // set const
-  pCreateTbReq->flags = 0;
-  pCreateTbReq->type = TSDB_CHILD_TABLE;
-  pCreateTbReq->ctb.suid = suid;
-
-  // set super table name
-  SName name = {0};
-  tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
 
   STagVal tagVal = { .cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
-  taosArrayPush(tagArray, &tagVal);
+  taosArrayPush(pTagArray, &tagVal);
-  pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
 
-  STag* pTag = NULL;
+  tTagNew(pTagArray, 1, false, (STag**) &pCreateTbReq->ctb.pTag);
-  tTagNew(tagArray, 1, false, &pTag);
-  taosArrayDestroy(tagArray);
 
-  if (pTag == NULL) {
+  if (pCreateTbReq->ctb.pTag == NULL) {
     tdDestroySVCreateTbReq(pCreateTbReq);
     taosMemoryFreeClear(pCreateTbReq);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  pCreateTbReq->ctb.pTag = (uint8_t*)pTag;
+  pCreateTbReq->ctb.tagName = createDefaultTagColName();
-
-  // set tag name
-  SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-  char k[TSDB_COL_NAME_LEN] = "group_id";
-  taosArrayPush(tagName, k);
-
-  pCreateTbReq->ctb.tagName = tagName;
 
   // set table name
-  pCreateTbReq->name = taosStrdup(ctbName);
+  setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId);
   return pCreateTbReq;
 }
 
@@ -514,7 +495,7 @@ int32_t tsAscendingSortFn(const void* p1, const void* p2) {
   }
 }
 
-int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
+int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
   int32_t numOfRows = pDataBlock->info.rows;
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -592,7 +573,7 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
   const char* id = pTask->id.idStr;
 
   while (pTableSinkInfo->uid == 0) {
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask)) {
       tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName);
       return TSDB_CODE_STREAM_EXEC_CANCELLED;
     }
@@ -693,8 +674,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
 
       tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName);
 
+      SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
+
       pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
-      pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock);
+      pTableData->pCreateTbReq =
+          buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray);
+      taosArrayDestroy(pTagArray);
+
       if (pTableData->pCreateTbReq == NULL) {
         tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno));
         taosMemoryFree(pTableSinkInfo);
@@ -724,17 +710,16 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock,
+int32_t setDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
-                               SSubmitTbData* pTableData) {
+                               SSubmitTbData* pTableData, const char* id) {
-  int32_t     numOfRows = pDataBlock->info.rows;
+  int32_t numOfRows = pDataBlock->info.rows;
-  const char* id = pTask->id.idStr;
 
   tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64,
-          id, blockIndex + 1, numOfRows, pTask->outputInfo.tbSink.stbUid);
+          id, blockIndex + 1, numOfRows, suid);
   char* dstTableName = pDataBlock->info.parTbName;
 
   // convert all rows
-  int32_t code = doConvertRows(pTableData, pTask->outputInfo.tbSink.pTSchema, pDataBlock, id);
+  int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, id);
   if (code != TSDB_CODE_SUCCESS) {
     tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno));
     return code;
@@ -773,7 +758,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
             numOfBlocks);
 
     for(int32_t i = 0; i < numOfBlocks; ++i) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
         return;
       }
 
@@ -800,7 +785,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           continue;
         }
 
-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }
@@ -823,7 +808,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
 
     bool hasSubmit = false;
     for (int32_t i = 0; i < numOfBlocks; i++) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
         taosHashCleanup(pTableIndexMap);
         tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
         return;
@@ -847,7 +832,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           continue;
         }
 
-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }
@@ -857,7 +842,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
         int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
         taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
       } else {
-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }

source/dnode/vnode/src/tq/tqStreamTask.c

@@ -60,7 +60,7 @@ int32_t tqScanWal(STQ* pTq) {
   return 0;
 }
 
-int32_t tqCheckAndRunStreamTask(STQ* pTq) {
+int32_t tqStartStreamTask(STQ* pTq) {
   int32_t      vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
 
@@ -92,19 +92,19 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
     }
 
     if (pTask->status.downstreamReady == 1) {
-      tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task",
+      if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
-              pTask->id.idStr);
+        tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task",
-      streamLaunchFillHistoryTask(pTask);
+                pTask->id.idStr);
+        streamLaunchFillHistoryTask(pTask);
+      }
+
+      streamMetaUpdateTaskReadyInfo(pTask);
       streamMetaReleaseTask(pMeta, pTask);
       continue;
     }
 
-    pTask->execInfo.init = taosGetTimestampMs();
+    EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
-    tqDebug("s-task:%s start check downstream tasks, set the init ts:%"PRId64, pTask->id.idStr, pTask->execInfo.init);
+    streamTaskHandleEvent(pTask->status.pSM, event);
-
-    streamSetStatusNormal(pTask);
-    streamTaskCheckDownstream(pTask);
-
     streamMetaReleaseTask(pMeta, pTask);
   }
 
@@ -112,9 +112,9 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
   return 0;
 }
 
-int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq) {
+int32_t tqLaunchStreamTaskAsync(STQ* pTq) {
-  int32_t      vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
+  int32_t      vgId = pMeta->vgId;
 
   int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
   if (numOfTasks == 0) {
@@ -228,7 +228,7 @@ int32_t tqStopStreamTasks(STQ* pTq) {
   return 0;
 }
 
-int32_t tqStartStreamTasks(STQ* pTq) {
+int32_t tqResetStreamTaskStatus(STQ* pTq) {
   SStreamMeta* pMeta = pTq->pStreamMeta;
   int32_t      vgId = TD_VID(pTq->pVnode);
   int32_t      numOfTasks = taosArrayGetSize(pMeta->pTaskList);
@@ -243,11 +243,7 @@ int32_t tqStartStreamTasks(STQ* pTq) {
 
     STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId};
     SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
-
+    streamTaskResetStatus(*pTask);
-    int8_t status = (*pTask)->status.taskStatus;
-    if (status == TASK_STATUS__STOP && (*pTask)->info.fillHistory != 1) {
-      streamSetStatusNormal(*pTask);
-    }
   }
 
   return 0;
@@ -328,15 +324,17 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
   }
 
   // not in ready state, do not handle the data from wal
-  int32_t status = pTask->status.taskStatus;
+//  int32_t status = pTask->status.taskStatus;
-  if (status != TASK_STATUS__NORMAL) {
+  char* p = NULL;
-    tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
+  int32_t status = streamTaskGetStatus(pTask, &p);
+  if (streamTaskGetStatus(pTask, &p) != TASK_STATUS__READY) {
+    tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, p);
     return false;
   }
 
   // fill-history task has entered into the last phase, no need to anything
   if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
-    ASSERT(status == TASK_STATUS__NORMAL);
+    ASSERT(status == TASK_STATUS__READY);
     // the maximum version of data in the WAL has reached already, the step2 is done
     tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
             pTask->dataRange.range.maxVer);
@@ -344,13 +342,13 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
   }
 
   // check if input queue is full or not
-  if (streamQueueIsFull(pTask->inputInfo.queue)) {
+  if (streamQueueIsFull(pTask->inputq.queue)) {
     tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr);
     return false;
   }
 
   // the input queue of downstream task is full, so the output is blocked, stopped for a while
-  if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+  if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
     tqDebug("s-task:%s inputQ is blocked, do nothing", pTask->id.idStr);
     return false;
   }
@@ -444,14 +442,15 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
       continue;
     }
 
-    int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputInfo.queue);
+    int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
     int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX;
 
     taosThreadMutexLock(&pTask->lock);
 
-    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+    char* p = NULL;
-    if (pTask->status.taskStatus != TASK_STATUS__NORMAL) {
+    ETaskStatus status = streamTaskGetStatus(pTask, &p);
-      tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pStatus);
+    if (status != TASK_STATUS__READY) {
+      tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, p);
       taosThreadMutexUnlock(&pTask->lock);
       streamMetaReleaseTask(pStreamMeta, pTask);
       continue;

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -1200,11 +1200,28 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
   }
 }
 
-static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SFileDataBlockInfo* pBlock) {
+static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, SLastBlockReader* pLastBlockReader, int32_t order) {
   bool ascScan = ASCENDING_TRAVERSE(order);
+  bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);
 
-  return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->record.firstKey)) ||
+  int64_t key = 0;
-         (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->record.lastKey));
+  if (bHasDataInLastBlock) {
+    int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+    key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt);
+  } else {
+    key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey;
+  }
+
+  return key;
+}
+
+static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock,
+                                     SLastBlockReader* pLastBlockReader, int32_t order) {
+  bool    ascScan = ASCENDING_TRAVERSE(order);
+  int64_t key = getBoarderKeyInFiles(pBlock, pLastBlockReader, order);
+
+  return (ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts < key)) ||
+         (!ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts > key));
 }
 
 static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) {
@@ -2667,6 +2684,15 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
   }
 }
 
+static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, SLastBlockReader* pLastBlockReader, bool asc) {
+  if(!hasDataInLastBlock(pLastBlockReader)) {
+    return true;
+  } else {
+    int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+    return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt);
+  }
+}
+
 static int32_t doBuildDataBlock(STsdbReader* pReader) {
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -2702,17 +2728,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
 
     // build composed data block
     code = buildComposedDataBlock(pReader);
-  } else if (bufferDataInFileBlockGap(pReader->info.order, keyInBuf, pBlockInfo)) {
+  } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pLastBlockReader, pReader->info.order)) {
-    // data in memory that are earlier than current file block
+    // data in memory that are earlier than current file block and stt blocks
     // rows in buffer should be less than the file block in asc, greater than file block in desc
-    int64_t endKey =
+    int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pLastBlockReader, pReader->info.order);
-        (ASCENDING_TRAVERSE(pReader->info.order)) ? pBlockInfo->record.firstKey : pBlockInfo->record.lastKey;
     code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
   } else {
-    bool    bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);
+    if (notOverlapWithSttFiles(pBlockInfo, pLastBlockReader, asc)) {
-    int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN;
-    if (!bHasDataInLastBlock ||
-        ((asc && pBlockInfo->record.lastKey < tsLast) || (!asc && pBlockInfo->record.firstKey > tsLast))) {
       // whole block is required, return it directly
       SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
       pInfo->rows = pBlockInfo->record.numRow;
@@ -2723,7 +2745,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
       setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
 
       // update the last key for the corresponding table
-      pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? pInfo->window.ekey : pInfo->window.skey;
+      pScanInfo->lastKey = asc ? pInfo->window.ekey : pInfo->window.skey;
       tsdbDebug("%p uid:%" PRIu64
                 " clean file block retrieved from file, global index:%d, "
                 "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
@@ -2750,10 +2772,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
         }
 
         // data in stt now overlaps with current active file data block, need to composed with file data block.
-        int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+        int64_t lastKeyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
-        if ((keyInStt >= pBlockInfo->record.firstKey && asc) || (keyInStt <= pBlockInfo->record.lastKey && (!asc))) {
+        if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) ||
-          tsdbDebug("%p keyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
+            (lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) {
-                    keyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
+          tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
+                    lastKeyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
           break;
         }
       }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -585,7 +585,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
       }
     } break;
     case TDMT_VND_STREAM_TASK_RESET: {
-      if (pVnode->restored/* && vnodeIsLeader(pVnode)*/) {
+      if (pVnode->restored && vnodeIsLeader(pVnode)) {
         tqProcessTaskResetReq(pVnode->pTq, pMsg);
       }
     } break;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -555,7 +555,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
   pVnode->restored = true;
 
   taosWLockLatch(&pVnode->pTq->pStreamMeta->lock);
-  if (pVnode->pTq->pStreamMeta->startInfo.startedAfterNodeUpdate) {
+  if (pVnode->pTq->pStreamMeta->startInfo.startAllTasksFlag) {
     vInfo("vgId:%d, sync restore finished, stream tasks will be launched by other thread", vgId);
     taosWUnLockLatch(&pVnode->pTq->pStreamMeta->lock);
     return;
@@ -567,8 +567,8 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
       vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId);
     } else {
       vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId);
-      tqStartStreamTasks(pVnode->pTq);
+      tqResetStreamTaskStatus(pVnode->pTq);
-      tqCheckAndRunStreamTaskAsync(pVnode->pTq);
+      tqLaunchStreamTaskAsync(pVnode->pTq);
     }
   } else {
     vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId);

source/libs/stream/inc/streamInt.h

@@ -52,6 +52,11 @@ extern "C" {
 #define stTrace(...) do { if (stDebugFlag & DEBUG_TRACE) { taosPrintLog("STM ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
+typedef struct {
+  int8_t       type;
+  SSDataBlock* pBlock;
+} SStreamTrigger;
+
 typedef struct SStreamGlobalEnv {
   int8_t inited;
   void*  timer;
@@ -122,7 +127,7 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask);
 int32_t streamTaskFillHistoryFinished(SStreamTask* pTask);
 int32_t streamTransferStateToStreamTask(SStreamTask* pTask);
 
-int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, int32_t quotaRate);
+int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate);
 STaskId streamTaskExtractKey(const SStreamTask* pTask);
 void    streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo);
 void    streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo);

source/libs/stream/inc/streamsm.h

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_STREAMSM_H
+#define TDENGINE_STREAMSM_H
+
+#include "tstream.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// moore finite state machine for stream task
+typedef struct SStreamTaskState {
+  ETaskStatus state;
+  char*       name;
+} SStreamTaskState;
+
+typedef int32_t (*__state_trans_fn)(SStreamTask*);
+typedef int32_t (*__state_trans_succ_fn)(SStreamTask*);
+
+typedef struct SAttachedEventInfo {
+  ETaskStatus      status;    // required status that this event can be handled
+  EStreamTaskEvent event;     // the delayed handled event
+} SAttachedEventInfo;
+
+typedef struct STaskStateTrans {
+  bool                  autoInvokeEndFn;
+  SStreamTaskState      state;
+  EStreamTaskEvent      event;
+  SStreamTaskState      next;
+  __state_trans_fn      pAction;
+  __state_trans_succ_fn pSuccAction;
+  SAttachedEventInfo    attachEvent;
+} STaskStateTrans;
+
+struct SStreamTaskSM {
+  SStreamTask*     pTask;
+  STaskStateTrans* pActiveTrans;
+  int64_t          startTs;
+  SStreamTaskState current;
+  struct {
+    SStreamTaskState state;
+    EStreamTaskEvent evt;
+  } prev;
+  // register the next handled event, if current state is not allowed to handle this event
+  SArray* pWaitingEventList;
+};
+
+typedef struct SStreamEventInfo {
+  EStreamTaskEvent event;
+  const char*      name;
+} SStreamEventInfo;
+
+SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
+void* streamDestroyStateMachine(SStreamTaskSM* pSM);
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_STREAMSM_H

source/libs/stream/src/stream.c

@@ -62,14 +62,14 @@ static void streamSchedByTimer(void* param, void* tmrId) {
   int32_t      nextTrigger = (int32_t)pTask->info.triggerParam;
 
   int8_t status = atomic_load_8(&pTask->schedInfo.status);
-  stDebug("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
+  stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
 
-  if (streamTaskShouldStop(&pTask->status) || streamTaskShouldPause(&pTask->status)) {
+  if (streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) {
     stDebug("s-task:%s jump out of schedTimer", id);
     return;
   }
 
-  if (pTask->status.taskStatus == TASK_STATUS__CK) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) {
     stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger);
   } else {
     if (status == TASK_TRIGGER_STATUS__ACTIVE) {
@@ -127,7 +127,7 @@ int32_t streamSchedExec(SStreamTask* pTask) {
     SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
     if (pRunReq == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      /*int8_t status = */streamTaskSetSchedStatusInActive(pTask);
+      /*int8_t status = */streamTaskSetSchedStatusInactive(pTask);
       stError("failed to create msg to aunch s-task:%s, reason out of memory", pTask->id.idStr);
       return -1;
     }
@@ -267,8 +267,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
   }
 
   // disable the data from upstream tasks
-  int8_t st = pTask->status.taskStatus;
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__HALT) {
-  if (st == TASK_STATUS__HALT) {
     status = TASK_INPUT_STATUS__BLOCKED;
   }
 
@@ -297,7 +296,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S
   return 0;
 }
 
-void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputInfo.status, TASK_INPUT_STATUS__FAILED); }
+void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); }
 
 void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) {
   int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);

source/libs/stream/src/streamBackendRocksdb.c

@@ -993,7 +993,7 @@ int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) {
       stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pHandle, pChkpIdDir,
               taosGetTimestampMs() - st);
     }
-  } else  {
+  } else {
     stError("stream backend:%p failed to flush db at:%s", pHandle, pChkpIdDir);
   }
   // release all ref to cfWrapper;
@@ -1467,16 +1467,24 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_c
 
 void destroyRocksdbCfInst(RocksdbCfInst* inst) {
   int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]);
-  for (int i = 0; i < cfLen; i++) {
+  if (inst->pHandle) {
-    if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]);
+    for (int i = 0; i < cfLen; i++) {
+      if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]);
+    }
+    taosMemoryFree(inst->pHandle);
   }
 
-  rocksdb_writeoptions_destroy(inst->wOpt);
+  if (inst->cfOpt) {
-  inst->wOpt = NULL;
+    for (int i = 0; i < cfLen; i++) {
+      rocksdb_options_destroy(inst->cfOpt[i]);
+      rocksdb_block_based_options_destroy(((RocksdbCfParam*)inst->param)[i].tableOpt);
+    }
+    taosMemoryFreeClear(inst->cfOpt);
+    taosMemoryFreeClear(inst->param);
+  }
+  if (inst->wOpt) rocksdb_writeoptions_destroy(inst->wOpt);
+  if (inst->rOpt) rocksdb_readoptions_destroy(inst->rOpt);
 
-  rocksdb_readoptions_destroy(inst->rOpt);
-  taosMemoryFree(inst->cfOpt);
-  taosMemoryFreeClear(inst->param);
   taosMemoryFree(inst);
 }
 
@@ -1645,6 +1653,13 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
     pState->pTdbState->backendCfWrapperId = id;
     pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper;
     stInfo("succ to open state %p on backendWrapper, %p, %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr);
+
+    inst->pHandle = NULL;
+    inst->cfOpt = NULL;
+    inst->param = NULL;
+
+    inst->wOpt = NULL;
+    inst->rOpt = NULL;
     return 0;
   }
   taosThreadMutexUnlock(&handle->cfMutex);
@@ -1992,14 +2007,12 @@ int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* ke
   return 0;
 }
 int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur) {
-  stDebug("streamStateCurPrev_rocksdb");
   if (!pCur) return -1;
 
   rocksdb_iter_prev(pCur->iter);
   return 0;
 }
 int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
-  stDebug("streamStateGetKVByCur_rocksdb");
   if (!pCur) return -1;
   SStateKey  tkey;
   SStateKey* pKtmp = &tkey;
@@ -2024,7 +2037,6 @@ int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, cons
   return -1;
 }
 SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key) {
-  stDebug("streamStateGetAndCheckCur_rocksdb");
   SStreamStateCur* pCur = streamStateFillGetCur_rocksdb(pState, key);
   if (pCur) {
     int32_t code = streamStateGetGroupKVByCur_rocksdb(pCur, key, NULL, 0);
@@ -2035,7 +2047,6 @@ SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey
 }
 
 SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) {
-  stDebug("streamStateSeekKeyNext_rocksdb");
   SStreamStateCur* pCur = createStreamStateCursor();
   if (pCur == NULL) {
     return NULL;
@@ -2074,7 +2085,6 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin
 }
 
 SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
-  stDebug("streamStateGetCur_rocksdb");
   int32_t code = 0;
 
   const SStateKey maxStateKey = {.key = {.groupId = UINT64_MAX, .ts = INT64_MAX}, .opNum = INT64_MAX};
@@ -2115,7 +2125,6 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
 }
 
 SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) {
-  stDebug("streamStateGetCur_rocksdb");
   SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper;
 
   SStreamStateCur* pCur = createStreamStateCursor();

source/libs/stream/src/streamCheckpoint.c

@@ -135,12 +135,13 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) {
   ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
 
+  // todo this status may not be set here.
   // 1. set task status to be prepared for check point, no data are allowed to put into inputQ.
-  pTask->status.taskStatus = TASK_STATUS__CK;
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
+
   pTask->checkpointingId = pReq->checkpointId;
   pTask->checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask);
   pTask->chkInfo.startTs = taosGetTimestampMs();
-
   pTask->execInfo.checkpoint += 1;
 
   // 2. Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task
@@ -171,11 +172,15 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
   const char* id = pTask->id.idStr;
   int32_t code = TSDB_CODE_SUCCESS;
 
-  // set the task status
-  pTask->checkpointingId = checkpointId;
-
   // set task status
-  pTask->status.taskStatus = TASK_STATUS__CK;
+  if (streamTaskGetStatus(pTask, NULL) != TASK_STATUS__CK) {
+    pTask->checkpointingId = checkpointId;
+    code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
+    if (code != TSDB_CODE_SUCCESS) {
+      stError("s-task:%s handle checkpoint-trigger block failed, code:%s", id, tstrerror(code));
+      return code;
+    }
+  }
 
   { // todo: remove this when the pipeline checkpoint generating is used.
     SStreamMeta* pMeta = pTask->pMeta;
@@ -195,6 +200,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
       stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
       continueDispatchCheckpointBlock(pBlock, pTask);
     } else {  // only one task exists, no need to dispatch downstream info
+      atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1);
       streamProcessCheckpointReadyMsg(pTask);
       streamFreeQitem((SStreamQueueItem*)pBlock);
     }
@@ -274,6 +280,8 @@ void streamTaskClearCheckInfo(SStreamTask* pTask) {
 }
 
 int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) {
+  int32_t vgId = pMeta->vgId;
+
   taosWLockLatch(&pMeta->lock);
 
   for (int32_t i = 0; i < taosArrayGetSize(pMeta->pTaskList); ++i) {
@@ -288,27 +296,34 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) {
       continue;
     }
 
-    int8_t prev = p->status.taskStatus;
     ASSERT(p->chkInfo.checkpointId < p->checkpointingId && p->checkpointingId == checkpointId);
 
     p->chkInfo.checkpointId = p->checkpointingId;
     streamTaskClearCheckInfo(p);
-    streamSetStatusNormal(p);
 
-    // save the task
+    char* str = NULL;
-    streamMetaSaveTask(pMeta, p);
+    streamTaskGetStatus(p, &str);
+
+    int32_t code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
+    if (code != TSDB_CODE_SUCCESS) {
+      stDebug("s-task:%s vgId:%d save task status failed, since handle event failed", p->id.idStr, vgId);
+      taosWUnLockLatch(&pMeta->lock);
+      return -1;
+    } else { // save the task
+      streamMetaSaveTask(pMeta, p);
+    }
 
     stDebug(
         "vgId:%d s-task:%s level:%d open upstream inputQ, commit task status after checkpoint completed, "
         "checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s",
         pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer,
-        streamGetTaskStatusStr(prev));
+        str);
   }
 
   if (streamMetaCommit(pMeta) < 0) {
     taosWUnLockLatch(&pMeta->lock);
     stError("vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s", pMeta->vgId,
-           checkpointId, terrstr());
+            checkpointId, terrstr());
     return -1;
   } else {
     taosWUnLockLatch(&pMeta->lock);

source/libs/stream/src/streamDispatch.c

@@ -420,7 +420,7 @@ static void doRetryDispatchData(void* param, void* tmrId) {
   const char*  id = pTask->id.idStr;
   int32_t      msgId = pTask->execInfo.dispatch;
 
-  if (streamTaskShouldStop(&pTask->status)) {
+  if (streamTaskShouldStop(pTask)) {
     int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
     stDebug("s-task:%s should stop, abort from timer, ref:%d", pTask->id.idStr, ref);
     return;
@@ -474,10 +474,10 @@ static void doRetryDispatchData(void* param, void* tmrId) {
   }
 
   if (code != TSDB_CODE_SUCCESS) {
-    if (!streamTaskShouldStop(&pTask->status)) {
+    if (!streamTaskShouldStop(pTask)) {
 //      stDebug("s-task:%s reset the waitRspCnt to be 0 before launch retry dispatch", pTask->id.idStr);
 //      atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0);
-      if (streamTaskShouldPause(&pTask->status)) {
+      if (streamTaskShouldPause(pTask)) {
         streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS * 10);
       } else {
         streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS);
@@ -662,8 +662,10 @@ int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) {
     int32_t numOfVgs = taosArrayGetSize(vgInfo);
     pTask->notReadyTasks = numOfVgs;
 
+    char* p = NULL;
+    streamTaskGetStatus(pTask, &p);
     stDebug("s-task:%s send scan-history data complete msg to downstream (shuffle-dispatch) %d tasks, status:%s", pTask->id.idStr,
-           numOfVgs, streamGetTaskStatusStr(pTask->status.taskStatus));
+           numOfVgs, p);
     for (int32_t i = 0; i < numOfVgs; i++) {
       SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
       req.downstreamTaskId = pVgInfo->taskId;
@@ -775,8 +777,9 @@ int32_t doDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHist
   initRpcMsg(&msg, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, buf, tlen + sizeof(SMsgHead));
 
   tmsgSendReq(pEpSet, &msg);
-  const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+  char* p = NULL;
-  stDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, pStatus,
+  streamTaskGetStatus(pTask, &p);
+  stDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, p,
          pReq->downstreamTaskId, vgId);
   return 0;
 }
@@ -1008,7 +1011,7 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo,
   taosThreadMutexUnlock(&pTask->lock);
 
   int32_t num = taosArrayGetSize(pTask->pRspMsgList);
-  stDebug("s-task:%s add scan history finish rsp msg for task:0x%x, total:%d", pTask->id.idStr, pReq->upstreamTaskId,
+  stDebug("s-task:%s add scan-history finish rsp msg for task:0x%x, total:%d", pTask->id.idStr, pReq->upstreamTaskId,
          num);
   return TSDB_CODE_SUCCESS;
 }
@@ -1024,7 +1027,7 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask) {
     SStreamContinueExecInfo* pInfo = taosArrayGet(pTask->pRspMsgList, i);
     tmsgSendRsp(&pInfo->msg);
 
-    stDebug("s-task:%s level:%d notify upstream:0x%x continuing scan data in WAL", id, level, pInfo->taskId);
+    stDebug("s-task:%s level:%d notify upstream:0x%x continuing handle data in WAL", id, level, pInfo->taskId);
   }
 
   taosArrayClear(pTask->pRspMsgList);
@@ -1040,8 +1043,8 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId
   int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs;
 
   // put data into inputQ of current task is also allowed
-  if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+  if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
-    pTask->inputInfo.status = TASK_INPUT_STATUS__NORMAL;
+    pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
     stDebug("s-task:%s downstream task:0x%x resume to normal from inputQ blocking, blocking time:%" PRId64 "ms",
             pTask->id.idStr, downstreamId, el);
   } else {
@@ -1093,7 +1096,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
 
   } else {  // code == 0
     if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) {
-      pTask->inputInfo.status = TASK_INPUT_STATUS__BLOCKED;
+      pTask->inputq.status = TASK_INPUT_STATUS__BLOCKED;
       // block the input of current task, to push pressure to upstream
       taosThreadMutexLock(&pTask->lock);
       taosArrayPush(pTask->msgInfo.pRetryList, &pRsp->downstreamNodeId);
@@ -1113,16 +1116,16 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
     ASSERT(leftRsp >= 0);
 
     if (leftRsp > 0) {
-      stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%d, waiting for %d rsp",
+      stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, waiting for %d rsp",
-               id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, code, leftRsp);
+               id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), leftRsp);
     } else {
       stDebug(
-          "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%d, all rsp",
+          "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all rsp",
-          id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, code);
+          id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code));
     }
   } else {
-    stDebug("s-task:%s recv fix-dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%d",
+    stDebug("s-task:%s recv fix-dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s",
-            id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, code);
+            id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code));
   }
 
   ASSERT(leftRsp >= 0);
@@ -1181,6 +1184,9 @@ int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpda
     if (tEncodeSEpSet(pEncoder, &pInfo->prevEp) < 0) return -1;
     if (tEncodeSEpSet(pEncoder, &pInfo->newEp) < 0) return -1;
   }
+
+  // todo this new attribute will be result in being incompatible with previous version
+  if (tEncodeI32(pEncoder, pMsg->transId) < 0) return -1;
   tEndEncode(pEncoder);
   return pEncoder->pos;
 }
@@ -1201,6 +1207,8 @@ int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg*
     taosArrayPush(pMsg->pNodeList, &info);
   }
 
+  if (tDecodeI32(pDecoder, &pMsg->transId) < 0) return -1;
+
   tEndDecode(pDecoder);
   return 0;
 }

source/libs/stream/src/streamExec.c

@@ -22,14 +22,13 @@
 
 static int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask);
 
-bool streamTaskShouldStop(const SStreamStatus* pStatus) {
+bool streamTaskShouldStop(const SStreamTask* pTask) {
-  int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus);
+  ETaskStatus s = streamTaskGetStatus(pTask, NULL);
-  return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING);
+  return (s == TASK_STATUS__STOP) || (s == TASK_STATUS__DROPPING);
 }
 
-bool streamTaskShouldPause(const SStreamStatus* pStatus) {
+bool streamTaskShouldPause(const SStreamTask* pTask) {
-  int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus);
+  return (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE);
-  return (status == TASK_STATUS__PAUSE);
 }
 
 static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBlock) {
@@ -102,12 +101,12 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i
       pRes = taosArrayInit(4, sizeof(SSDataBlock));
     }
 
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask)) {
       taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
       return 0;
     }
 
-    if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+    if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
       stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry exec task", pTask->id.idStr);
       taosMsleep(1000);
       continue;
@@ -198,7 +197,7 @@ int32_t streamScanHistoryData(SStreamTask* pTask) {
   qSetStreamOpOpen(exec);
 
   while (!finished) {
-    if (streamTaskShouldPause(&pTask->status)) {
+    if (streamTaskShouldPause(pTask)) {
       double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
       stDebug("s-task:%s paused from the scan-history task, elapsed time:%.2fsec", pTask->id.idStr, el);
       break;
@@ -213,12 +212,12 @@ int32_t streamScanHistoryData(SStreamTask* pTask) {
     int32_t size = 0;
     int32_t numOfBlocks = 0;
     while (1) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
         taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
         return 0;
       }
 
-      if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+      if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
         stDebug("s-task:%s inputQ is blocked, wait for 10sec and retry", pTask->id.idStr);
         taosMsleep(10000);
         continue;
@@ -309,21 +308,19 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
            pStreamTask->id.idStr);
   }
 
-  ASSERT(((pStreamTask->status.taskStatus == TASK_STATUS__STOP) ||
+  ETaskStatus status = streamTaskGetStatus(pStreamTask, NULL);
-          (pStreamTask->hTaskInfo.id.taskId == pTask->id.taskId)) &&
+  ASSERT(((status == TASK_STATUS__DROPPING) || (pStreamTask->hTaskInfo.id.taskId == pTask->id.taskId)) &&
          pTask->status.appendTranstateBlock == true);
 
   STimeWindow* pTimeWindow = &pStreamTask->dataRange.window;
 
-  // todo. the dropping status should be append to the status after the halt completed.
   // It must be halted for a source stream task, since when the related scan-history-data task start scan the history
   // for the step 2.
-  int8_t status = pStreamTask->status.taskStatus;
   if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING);
+    ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP);
   } else {
-    ASSERT(status == TASK_STATUS__NORMAL);
+    ASSERT(status == TASK_STATUS__READY|| status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP);
-    pStreamTask->status.taskStatus = TASK_STATUS__HALT;
+    streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
     stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr);
   }
 
@@ -333,13 +330,14 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
   // In case of sink tasks, no need to halt them.
   // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to
   // start the task state transfer procedure.
-  // When a task is idle with halt status, all data in inputQ are consumed.
+  char* p = NULL;
+  streamTaskGetStatus(pStreamTask, &p);
   if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     // update the scan data range for source task.
     stDebug("s-task:%s level:%d stream task window %" PRId64 " - %" PRId64 " update to %" PRId64 " - %" PRId64
-           ", status:%s, sched-status:%d",
+            ", status:%s, sched-status:%d",
-           pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN,
+            pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN,
-           pTimeWindow->ekey, streamGetTaskStatusStr(TASK_STATUS__NORMAL), pStreamTask->status.schedStatus);
+            pTimeWindow->ekey, p, pStreamTask->status.schedStatus);
   } else {
     stDebug("s-task:%s no need to update time window for non-source task", pStreamTask->id.idStr);
   }
@@ -354,7 +352,7 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
 
   // 3. resume the state of stream task, after this function, the stream task will run immidately. But it can not be
   // pause, since the pause allowed attribute is not set yet.
-  streamTaskResumeFromHalt(pStreamTask);
+  streamTaskResume(pStreamTask);  // todo refactor: use streamTaskResume.
 
   stDebug("s-task:%s fill-history task set status to be dropping, save the state into disk", pTask->id.idStr);
 
@@ -362,20 +360,21 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
   streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id);
 
   // 5. clear the link between fill-history task and stream task info
-  pStreamTask->hTaskInfo.id.taskId = 0;
+//  CLEAR_RELATED_FILLHISTORY_TASK(pStreamTask);
-  pStreamTask->hTaskInfo.id.streamId = 0;
 
   // 6. save to disk
   taosWLockLatch(&pMeta->lock);
-  streamMetaSaveTask(pMeta, pStreamTask);
+
-  if (streamMetaCommit(pMeta) < 0) {
+  pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask, NULL);
+//  streamMetaSaveTask(pMeta, pStreamTask);
+//  if (streamMetaCommit(pMeta) < 0) {
     // persist to disk
-  }
+//  }
   taosWUnLockLatch(&pMeta->lock);
 
   // 7. pause allowed.
   streamTaskEnablePause(pStreamTask);
-  if (taosQueueEmpty(pStreamTask->inputInfo.queue->pQueue)) {
+  if ((pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) && taosQueueEmpty(pStreamTask->inputq.queue->pQueue)) {
     SStreamRefDataBlock* pItem = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
 
     SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA);
@@ -492,7 +491,7 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock
       code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
       if (code == 0) {
         streamDispatchStreamBlock(pTask);
-      } else {
+      } else {  // todo put into queue failed, retry
         streamFreeQitem((SStreamQueueItem*)pBlock);
       }
     } else {  // level == TASK_LEVEL__SINK
@@ -500,12 +499,12 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock
     }
   } else {  // non-dispatch task, do task state transfer directly
     streamFreeQitem((SStreamQueueItem*)pBlock);
-    stDebug("s-task:%s non-dispatch task, start to transfer state directly", id);
+    stDebug("s-task:%s non-dispatch task, level:%d start to transfer state directly", id, pTask->info.taskLevel);
     ASSERT(pTask->info.fillHistory == 1);
-    code = streamTransferStateToStreamTask(pTask);
 
+    code = streamTransferStateToStreamTask(pTask);
     if (code != TSDB_CODE_SUCCESS) {
-      /*int8_t status = */ streamTaskSetSchedStatusInActive(pTask);
+      /*int8_t status = */ streamTaskSetSchedStatusInactive(pTask);
     }
   }
 
@@ -526,7 +525,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
     int32_t           blockSize = 0;
     int32_t           numOfBlocks = 0;
     SStreamQueueItem* pInput = NULL;
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__UNINIT)) {
       stDebug("s-task:%s stream task is stopped", id);
       break;
     }
@@ -592,8 +591,9 @@ int32_t streamExecForAll(SStreamTask* pTask) {
     // todo other thread may change the status
     // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
     if (type == STREAM_INPUT__CHECKPOINT) {
-      stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr,
+      char* p = NULL;
-              streamGetTaskStatusStr(pTask->status.taskStatus));
+      streamTaskGetStatus(pTask, &p);
+      stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p);
       streamTaskBuildCheckpoint(pTask);
       return 0;
     }
@@ -605,8 +605,15 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 // the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not
 // be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING.
 bool streamTaskIsIdle(const SStreamTask* pTask) {
-  return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || pTask->status.taskStatus == TASK_STATUS__STOP ||
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
-          pTask->status.taskStatus == TASK_STATUS__DROPPING);
+  return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || status == TASK_STATUS__STOP ||
+          status == TASK_STATUS__DROPPING);
+}
+
+bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) {
+  ETaskStatus st = streamTaskGetStatus(pTask, NULL);
+  return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__STREAM_SCAN_HISTORY ||
+          st == TASK_STATUS__CK);
 }
 
 int32_t streamExecTask(SStreamTask* pTask) {
@@ -623,20 +630,23 @@ int32_t streamExecTask(SStreamTask* pTask) {
       }
 
       taosThreadMutexLock(&pTask->lock);
-      if ((streamQueueGetNumOfItems(pTask->inputInfo.queue) == 0) || streamTaskShouldStop(&pTask->status) ||
+      if ((streamQueueGetNumOfItems(pTask->inputq.queue) == 0) || streamTaskShouldStop(pTask) ||
-          streamTaskShouldPause(&pTask->status)) {
+          streamTaskShouldPause(pTask)) {
         atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
         taosThreadMutexUnlock(&pTask->lock);
 
-        stDebug("s-task:%s exec completed, status:%s, sched-status:%d", id,
+        char* p = NULL;
-               streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus);
+        streamTaskGetStatus(pTask, &p);
+        stDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, p, pTask->status.schedStatus);
         return 0;
       }
       taosThreadMutexUnlock(&pTask->lock);
     }
   } else {
-    stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id,
+    char* p = NULL;
-           streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus);
+    streamTaskGetStatus(pTask, &p);
+    stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p,
+            pTask->status.schedStatus);
   }
 
   return 0;

source/libs/stream/src/streamMeta.c

@@ -143,8 +143,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     goto _err;
   }
 
-  pMeta->pUpdateTaskSet = taosHashInit(64, fp, false, HASH_NO_LOCK);
+  pMeta->updateInfo.pTasks = taosHashInit(64, fp, false, HASH_NO_LOCK);
-  if (pMeta->pUpdateTaskSet == NULL) {
+  if (pMeta->updateInfo.pTasks == NULL) {
     goto _err;
   }
 
@@ -211,7 +211,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
          stage);
   return pMeta;
 
-_err:
+  _err:
   taosMemoryFree(pMeta->path);
   if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap);
   if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList);
@@ -219,7 +219,7 @@ _err:
   if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb);
   if (pMeta->db) tdbClose(pMeta->db);
   if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo);
-  if (pMeta->pUpdateTaskSet) taosHashCleanup(pMeta->pUpdateTaskSet);
+  if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks);
   if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
   taosMemoryFree(pMeta);
 
@@ -228,11 +228,15 @@ _err:
 }
 
 int32_t streamMetaReopen(SStreamMeta* pMeta) {
+  // backup the restart flag
+  int32_t restartFlag = pMeta->startInfo.startAllTasksFlag;
   streamMetaClear(pMeta);
 
+  pMeta->startInfo.startAllTasksFlag = restartFlag;
+
+  // NOTE: role should not be changed during reopen meta
   pMeta->streamBackendRid = -1;
   pMeta->streamBackend = NULL;
-  pMeta->role = NODE_ROLE_UNINIT;
 
   char* defaultPath = taosMemoryCalloc(1, strlen(pMeta->path) + 128);
   sprintf(defaultPath, "%s%s%s", pMeta->path, TD_DIRSEP, "state");
@@ -274,6 +278,7 @@ int32_t streamMetaReopen(SStreamMeta* pMeta) {
 }
 
 void streamMetaClear(SStreamMeta* pMeta) {
+  // remove all existed tasks in this vnode
   void* pIter = NULL;
   while ((pIter = taosHashIterate(pMeta->pTasksMap, pIter)) != NULL) {
     SStreamTask* p = *(SStreamTask**)pIter;
@@ -299,6 +304,9 @@ void streamMetaClear(SStreamMeta* pMeta) {
   taosArrayClear(pMeta->chkpInUse);
   pMeta->numOfStreamTasks = 0;
   pMeta->numOfPausedTasks = 0;
+  pMeta->chkptNotReadyTasks = 0;
+
+  streamMetaResetStartInfo(&pMeta->startInfo);
 }
 
 void streamMetaClose(SStreamMeta* pMeta) {
@@ -336,7 +344,7 @@ void streamMetaCloseImpl(void* arg) {
 
   taosHashCleanup(pMeta->pTasksMap);
   taosHashCleanup(pMeta->pTaskBackendUnique);
-  taosHashCleanup(pMeta->pUpdateTaskSet);
+  taosHashCleanup(pMeta->updateInfo.pTasks);
   taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
 
   taosMemoryFree(pMeta->pHbInfo);
@@ -348,6 +356,7 @@ void streamMetaCloseImpl(void* arg) {
   stDebug("end to close stream meta");
 }
 
+// todo let's check the status for each task
 int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   void*   buf = NULL;
   int32_t len;
@@ -437,31 +446,13 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) {
   return (int32_t)size;
 }
 
-int32_t streamMetaGetNumOfStreamTasks(SStreamMeta* pMeta) {
-  int32_t num = 0;
-  size_t  size = taosArrayGetSize(pMeta->pTaskList);
-  for (int32_t i = 0; i < size; ++i) {
-    STaskId*      pId = taosArrayGet(pMeta->pTaskList, i);
-    SStreamTask** p = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId));
-    if (p == NULL) {
-      continue;
-    }
-
-    if ((*p)->info.fillHistory == 0) {
-      num += 1;
-    }
-  }
-
-  return num;
-}
-
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) {
   taosRLockLatch(&pMeta->lock);
 
   STaskId       id = {.streamId = streamId, .taskId = taskId};
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
   if (ppTask != NULL) {
-    if (!streamTaskShouldStop(&(*ppTask)->status)) {
+    if (!streamTaskShouldStop(*ppTask)) {
       int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
       taosRUnLockLatch(&pMeta->lock);
       stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref);
@@ -478,7 +469,7 @@ void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask)
   if (ref > 0) {
     stTrace("s-task:%s release task, ref:%d", pTask->id.idStr, ref);
   } else if (ref == 0) {
-    ASSERT(streamTaskShouldStop(&pTask->status));
+    ASSERT(streamTaskShouldStop(pTask));
     stTrace("s-task:%s all refs are gone, free it", pTask->id.idStr);
     tFreeStreamTask(pTask);
   } else if (ref < 0) {
@@ -506,11 +497,15 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
   if (ppTask) {
     pTask = *ppTask;
-    if (streamTaskShouldPause(&pTask->status)) {
+
+    // desc the paused task counter
+    if (streamTaskShouldPause(pTask)) {
       int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
       stInfo("vgId:%d s-task:%s drop stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
     }
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
+
+    // handle the dropping event
+    streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_DROPPING);
   } else {
     stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId);
     taosWUnLockLatch(&pMeta->lock);
@@ -518,13 +513,12 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
   }
   taosWUnLockLatch(&pMeta->lock);
 
-  stDebug("s-task:0x%x set task status:%s and start to unregister it", taskId,
+  stDebug("s-task:0x%x set task status:dropping and start to unregister it", taskId);
-          streamGetTaskStatusStr(TASK_STATUS__DROPPING));
 
   while (1) {
     taosRLockLatch(&pMeta->lock);
-    ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
 
+    ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
     if (ppTask) {
       if ((*ppTask)->status.timerActive == 0) {
         taosRUnLockLatch(&pMeta->lock);
@@ -544,23 +538,19 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
   taosWLockLatch(&pMeta->lock);
   ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
   if (ppTask) {
+    pTask = *ppTask;
+
     // it is an fill-history task, remove the related stream task's id that points to it
-    if ((*ppTask)->info.fillHistory == 1) {
+    if (pTask->info.fillHistory == 1) {
-      STaskId streamTaskId = {.streamId = (*ppTask)->streamTaskId.streamId, .taskId = (*ppTask)->streamTaskId.taskId};
+      streamTaskClearHTaskAttr(pTask);
-      SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &streamTaskId, sizeof(streamTaskId));
-      if (ppStreamTask != NULL) {
-        (*ppStreamTask)->hTaskInfo.id.taskId = 0;
-        (*ppStreamTask)->hTaskInfo.id.streamId = 0;
-      }
     } else {
       atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1);
     }
 
     taosHashRemove(pMeta->pTasksMap, &id, sizeof(id));
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
+    doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id);
 
     ASSERT(pTask->status.timerActive == 0);
-    doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id);
 
     if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) {
       stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt);
@@ -691,7 +681,8 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) {
       tFreeStreamTask(pTask);
       stError(
           "vgId:%d stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild "
-          "stream manually", vgId, tsDataDir);
+          "stream manually",
+          vgId, tsDataDir);
       return -1;
     }
     tDecoderClear(&decoder);
@@ -701,8 +692,8 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) {
       tFreeStreamTask(pTask);
 
       STaskId id = streamTaskExtractKey(pTask);
-
       taosArrayPush(pRecycleList, &id);
+
       int32_t total = taosArrayGetSize(pRecycleList);
       stDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total);
       continue;
@@ -738,7 +729,7 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) {
       atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1);
     }
 
-    if (streamTaskShouldPause(&pTask->status)) {
+    if (streamTaskShouldPause(pTask)) {
       atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
     }
 
@@ -790,6 +781,15 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
     if (tEncodeI64(pEncoder, ps->activeCheckpointId) < 0) return -1;
     if (tEncodeI8(pEncoder, ps->checkpointFailed) < 0) return -1;
   }
+
+  int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes);
+  if (tEncodeI32(pEncoder, numOfVgs) < 0) return -1;
+
+  for (int j = 0; j < numOfVgs; ++j) {
+    int32_t* pVgId = taosArrayGet(pReq->pUpdateNodes, j);
+    if (tEncodeI32(pEncoder, *pVgId) < 0) return -1;
+  }
+
   tEndEncode(pEncoder);
   return pEncoder->pos;
 }
@@ -823,6 +823,17 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
     taosArrayPush(pReq->pTaskStatus, &entry);
   }
 
+  int32_t numOfVgs = 0;
+  if (tDecodeI32(pDecoder, &numOfVgs) < 0) return -1;
+
+  pReq->pUpdateNodes = taosArrayInit(numOfVgs, sizeof(int32_t));
+
+  for (int j = 0; j < numOfVgs; ++j) {
+    int32_t vgId = 0;
+    if (tDecodeI32(pDecoder, &vgId) < 0) return -1;
+    taosArrayPush(pReq->pUpdateNodes, &vgId);
+  }
+
   tEndDecode(pDecoder);
   return 0;
 }
@@ -852,8 +863,8 @@ void metaHbToMnode(void* param, void* tmrId) {
   }
 
   // not leader not send msg
-  if (pMeta->role == NODE_ROLE_FOLLOWER) {
+  if (pMeta->role != NODE_ROLE_LEADER) {
-    stInfo("vgId:%d follower not send hb to mnode", pMeta->vgId);
+    stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role);
     taosReleaseRef(streamMetaId, rid);
     pMeta->pHbInfo->hbStart = 0;
     return;
@@ -877,13 +888,14 @@ void metaHbToMnode(void* param, void* tmrId) {
   int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
 
   SEpSet epset = {0};
-  bool   hasValEpset = false;
+  bool   hasMnodeEpset = false;
+
   hbMsg.vgId = pMeta->vgId;
   hbMsg.pTaskStatus = taosArrayInit(numOfTasks, sizeof(STaskStatusEntry));
+  hbMsg.pUpdateNodes = taosArrayInit(numOfTasks, sizeof(int32_t));
 
   for (int32_t i = 0; i < numOfTasks; ++i) {
-    STaskId* pId = taosArrayGet(pMeta->pTaskList, i);
+    STaskId*      pId = taosArrayGet(pMeta->pTaskList, i);
-
     SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId));
 
     // not report the status of fill-history task
@@ -893,10 +905,10 @@ void metaHbToMnode(void* param, void* tmrId) {
 
     STaskStatusEntry entry = {
         .id = *pId,
-        .status = (*pTask)->status.taskStatus,
+        .status = streamTaskGetStatus(*pTask, NULL),
         .nodeId = pMeta->vgId,
         .stage = pMeta->stage,
-        .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputInfo.queue)),
+        .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)),
     };
 
     entry.inputRate = entry.inputQUsed*100.0/STREAM_TASK_QUEUE_CAPACITY_IN_SIZE;
@@ -915,18 +927,39 @@ void metaHbToMnode(void* param, void* tmrId) {
       walReaderValidVersionRange((*pTask)->exec.pWalReader, &entry.verStart, &entry.verEnd);
     }
 
-    taosArrayPush(hbMsg.pTaskStatus, &entry);
+    taosThreadMutexLock(&(*pTask)->lock);
+    int32_t num = taosArrayGetSize((*pTask)->outputInfo.pDownstreamUpdateList);
+    for (int j = 0; j < num; ++j) {
+      int32_t *pNodeId = taosArrayGet((*pTask)->outputInfo.pDownstreamUpdateList, j);
 
-    if (!hasValEpset) {
+      bool exist = false;
+      int32_t numOfExisted = taosArrayGetSize(hbMsg.pUpdateNodes);
+      for (int k = 0; k < numOfExisted; ++k) {
+        if (*pNodeId == *(int32_t*)taosArrayGet(hbMsg.pUpdateNodes, k)) {
+          exist = true;
+          break;
+        }
+      }
+
+      if (!exist) {
+        taosArrayPush(hbMsg.pUpdateNodes, pNodeId);
+      }
+    }
+
+    taosArrayClear((*pTask)->outputInfo.pDownstreamUpdateList);
+    taosThreadMutexUnlock(&(*pTask)->lock);
+
+    taosArrayPush(hbMsg.pTaskStatus, &entry);
+    if (!hasMnodeEpset) {
       epsetAssign(&epset, &(*pTask)->info.mnodeEpset);
-      hasValEpset = true;
+      hasMnodeEpset = true;
     }
   }
 
   hbMsg.numOfTasks = taosArrayGetSize(hbMsg.pTaskStatus);
   taosRUnLockLatch(&pMeta->lock);
 
-  if (hasValEpset) {
+  if (hasMnodeEpset) {
     int32_t code = 0;
     int32_t tlen = 0;
 
@@ -971,13 +1004,14 @@ void metaHbToMnode(void* param, void* tmrId) {
   }
 
   taosArrayDestroy(hbMsg.pTaskStatus);
+  taosArrayDestroy(hbMsg.pUpdateNodes);
+
   taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamEnv.timer, &pMeta->pHbInfo->hbTmr);
   taosReleaseRef(streamMetaId, rid);
 }
 
-static bool hasStreamTaskInTimer(SStreamMeta* pMeta) {
+bool streamMetaTaskInTimer(SStreamMeta* pMeta) {
   bool inTimer = false;
-
   taosWLockLatch(&pMeta->lock);
 
   void* pIter = NULL;
@@ -1031,7 +1065,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) {
   stDebug("vgId:%d start to check all tasks", vgId);
   int64_t st = taosGetTimestampMs();
 
-  while (hasStreamTaskInTimer(pMeta)) {
+  while (streamMetaTaskInTimer(pMeta)) {
     stDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
     taosMsleep(100);
   }
@@ -1054,5 +1088,6 @@ void streamMetaInitForSnode(SStreamMeta* pMeta) {
 
 void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) {
   taosHashClear(pStartInfo->pReadyTaskSet);
-  pStartInfo->startedAfterNodeUpdate = 0;
+  pStartInfo->startAllTasksFlag = 0;
+  pStartInfo->readyTs = 0;
 }

source/libs/stream/src/streamQueue.c

@@ -18,7 +18,7 @@
 #define MAX_STREAM_EXEC_BATCH_NUM                 32
 #define MAX_SMOOTH_BURST_RATIO                    5     // 5 sec
 #define WAIT_FOR_DURATION                         40
-#define SINK_TASK_IDLE_DURATION                   200   // 200 ms
+#define OUTPUT_QUEUE_FULL_WAIT_DURATION           500   // 500 ms
 
 // todo refactor:
 // read data from input queue
@@ -119,17 +119,8 @@ int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue) {
   return numOfItems1 + numOfItems2;
 }
 
-int32_t streamQueueGetAvailableSpace(const SStreamQueue* pQueue, int32_t* availNum, double* availSize) {
-  int32_t num = streamQueueGetNumOfItems(pQueue);
-  *availNum = STREAM_TASK_QUEUE_CAPACITY - num;
-
-  *availSize = STREAM_TASK_QUEUE_CAPACITY_IN_SIZE - taosQueueMemorySize(pQueue->pQueue);
-  return 0;
-}
-
-// todo: fix it: data in Qall is not included here
 int32_t streamQueueGetItemSize(const SStreamQueue* pQueue) {
-  return taosQueueMemorySize(pQueue->pQueue);
+  return taosQueueMemorySize(pQueue->pQueue) + taosQallUnAccessedMemSize(pQueue->qall);
 }
 
 int32_t streamQueueItemGetSize(const SStreamQueueItem* pItem) {
@@ -173,12 +164,12 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
   }
 
   while (1) {
-    if (streamTaskShouldPause(&pTask->status) || streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldPause(pTask) || streamTaskShouldStop(pTask)) {
       stDebug("s-task:%s task should pause, extract input blocks:%d", pTask->id.idStr, *numOfBlocks);
       return TSDB_CODE_SUCCESS;
     }
 
-    SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputInfo.queue);
+    SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue);
     if (qItem == NULL) {
       if ((taskLevel == TASK_LEVEL__SOURCE  || taskLevel == TASK_LEVEL__SINK) && (++retryTimes) < MAX_RETRY_TIMES) {
         taosMsleep(WAIT_FOR_DURATION);
@@ -220,7 +211,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
           streamTaskConsumeQuota(pTask->outputInfo.pTokenBucket, *blockSize);
         }
 
-        streamQueueProcessFail(pTask->inputInfo.queue);
+        streamQueueProcessFail(pTask->inputq.queue);
         return TSDB_CODE_SUCCESS;
       }
     } else {
@@ -241,7 +232,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
             streamTaskConsumeQuota(pTask->outputInfo.pTokenBucket, *blockSize);
           }
 
-          streamQueueProcessFail(pTask->inputInfo.queue);
+          streamQueueProcessFail(pTask->inputq.queue);
           return TSDB_CODE_SUCCESS;
         }
 
@@ -249,7 +240,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
       }
 
       *numOfBlocks += 1;
-      streamQueueProcessSuccess(pTask->inputInfo.queue);
+      streamQueueProcessSuccess(pTask->inputq.queue);
 
       if (*numOfBlocks >= MAX_STREAM_EXEC_BATCH_NUM) {
         stDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM);
@@ -267,12 +258,12 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
 
 int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) {
   int8_t      type = pItem->type;
-  STaosQueue* pQueue = pTask->inputInfo.queue->pQueue;
+  STaosQueue* pQueue = pTask->inputq.queue->pQueue;
-  int32_t     total = streamQueueGetNumOfItems(pTask->inputInfo.queue) + 1;
+  int32_t     total = streamQueueGetNumOfItems(pTask->inputq.queue) + 1;
 
   if (type == STREAM_INPUT__DATA_SUBMIT) {
     SStreamDataSubmit* px = (SStreamDataSubmit*)pItem;
-    if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pTask->inputInfo.queue)) {
+    if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pTask->inputq.queue)) {
       double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));
       stTrace(
           "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data",
@@ -299,7 +290,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
            msgLen, ver, total, size + SIZE_IN_MiB(msgLen));
   } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE ||
              type == STREAM_INPUT__REF_DATA_BLOCK) {
-    if (streamQueueIsFull(pTask->inputInfo.queue)) {
+    if (streamQueueIsFull(pTask->inputq.queue)) {
       double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));
 
       stTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort",
@@ -354,7 +345,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
   STaosQueue* pQueue = pTask->outputq.queue->pQueue;
 
   while (streamQueueIsFull(pTask->outputq.queue)) {
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask)) {
       stInfo("s-task:%s discard result block due to task stop", pTask->id.idStr);
       return TSDB_CODE_STREAM_EXEC_CANCELLED;
     }
@@ -362,9 +353,10 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
     int32_t total = streamQueueGetNumOfItems(pTask->outputq.queue);
     double  size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));
     // let's wait for there are enough space to hold this result pBlock
-    stDebug("s-task:%s outputQ is full, wait for 500ms and retry, outputQ items:%d, size:%.2fMiB", pTask->id.idStr,
+    stDebug("s-task:%s outputQ is full, wait for %dms and retry, outputQ items:%d, size:%.2fMiB", pTask->id.idStr,
-           total, size);
+            OUTPUT_QUEUE_FULL_WAIT_DURATION, total, size);
-    taosMsleep(500);
+
+    taosMsleep(OUTPUT_QUEUE_FULL_WAIT_DURATION);
   }
 
   int32_t code = taosWriteQitem(pQueue, pBlock);
@@ -381,7 +373,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, int32_t quotaRate) {
+int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate) {
   if (numCap < 10 || numRate < 10 || pBucket == NULL) {
     stError("failed to init sink task bucket, cap:%d, rate:%d", numCap, numRate);
     return TSDB_CODE_INVALID_PARA;

source/libs/stream/src/streamStart.c

@@ -17,6 +17,7 @@
 #include "trpc.h"
 #include "ttimer.h"
 #include "wal.h"
+#include "streamsm.h"
 
 typedef struct SLaunchHTaskInfo {
   SStreamMeta* pMeta;
@@ -34,16 +35,18 @@ static void              streamTaskSetRangeStreamCalc(SStreamTask* pTask);
 static int32_t           initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated);
 static SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 static void              tryLaunchHistoryTask(void* param, void* tmrId);
+static void              doProcessDownstreamReadyRsp(SStreamTask* pTask);
 
-static void streamTaskSetReady(SStreamTask* pTask, int32_t numOfReqs) {
+int32_t streamTaskSetReady(SStreamTask* pTask) {
-  SStreamMeta* pMeta = pTask->pMeta;
+  char*       p = NULL;
-  int32_t      vgId = pMeta->vgId;
+  int32_t     numOfDowns = streamTaskGetNumOfDownstream(pTask);
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
 
-  if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY && pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
+  if ((status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY) &&
+      pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
     pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->upstreamInfo.pList);
     stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s",
-           pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream,
+           pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, p);
-           streamGetTaskStatusStr(pTask->status.taskStatus));
   }
 
   ASSERT(pTask->status.downstreamReady == 0);
@@ -52,34 +55,10 @@ static void streamTaskSetReady(SStreamTask* pTask, int32_t numOfReqs) {
   pTask->execInfo.start = taosGetTimestampMs();
   int64_t el = (pTask->execInfo.start - pTask->execInfo.init);
   stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s",
-          pTask->id.idStr, numOfReqs, el, streamGetTaskStatusStr(pTask->status.taskStatus));
+          pTask->id.idStr, numOfDowns, el, p);
 
-  taosWLockLatch(&pMeta->lock);
+  streamMetaUpdateTaskReadyInfo(pTask);
-
+  return TSDB_CODE_SUCCESS;
-  STaskId id = streamTaskExtractKey(pTask);
-  taosHashPut(pMeta->startInfo.pReadyTaskSet, &id, sizeof(id), NULL, 0);
-
-  int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta);
-
-  if (taosHashGetSize(pMeta->startInfo.pReadyTaskSet) == numOfTotal) {
-    STaskStartInfo* pStartInfo = &pMeta->startInfo;
-    pStartInfo->readyTs = pTask->execInfo.start;
-
-    if (pStartInfo->startTs != 0) {
-      pStartInfo->elapsedTime = pStartInfo->readyTs - pStartInfo->startTs;
-    } else {
-      pStartInfo->elapsedTime = 0;
-    }
-
-    streamMetaResetStartInfo(pStartInfo);
-
-    stDebug("vgId:%d all %d task(s) are started successfully, last ready task:%s level:%d, startTs:%" PRId64
-            ", readyTs:%" PRId64 " total elapsed time:%.2fs",
-            vgId, numOfTotal, pTask->id.idStr, pTask->info.taskLevel, pStartInfo->startTs, pStartInfo->readyTs,
-            pStartInfo->elapsedTime / 1000.0);
-  }
-
-  taosWUnLockLatch(&pMeta->lock);
 }
 
 int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) {
@@ -114,33 +93,28 @@ static int32_t doStartScanHistoryTask(SStreamTask* pTask) {
 }
 
 int32_t streamTaskStartScanHistory(SStreamTask* pTask) {
-  ASSERT(pTask->status.downstreamReady == 1);
+  int32_t level = pTask->info.taskLevel;
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
 
-  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+  ASSERT(pTask->status.downstreamReady == 1 &&
-    if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+         ((status == TASK_STATUS__SCAN_HISTORY) || (status == TASK_STATUS__STREAM_SCAN_HISTORY)));
-      return doStartScanHistoryTask(pTask);
+
-    } else {
+  if (level == TASK_LEVEL__SOURCE) {
-      ASSERT(pTask->status.taskStatus == TASK_STATUS__NORMAL);
+    return doStartScanHistoryTask(pTask);
-      stDebug("s-task:%s no need to scan-history data, status:%s, sched-status:%d, ver:%" PRId64, pTask->id.idStr,
+  } else if (level == TASK_LEVEL__AGG) {
-              streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus,
-              walReaderGetCurrentVer(pTask->exec.pWalReader));
-      streamTaskEnablePause(pTask);
-    }
-  } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
     if (pTask->info.fillHistory) {
       streamSetParamForScanHistory(pTask);
       streamTaskEnablePause(pTask);
     }
-  } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
+  } else if (level == TASK_LEVEL__SINK) {
-    if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+    stDebug("s-task:%s sink task do nothing to handle scan-history", pTask->id.idStr);
-      stDebug("s-task:%s sink task do nothing to handle scan-history", pTask->id.idStr);
-    }
   }
+
   return 0;
 }
 
 // check status
-static int32_t doCheckDownstreamStatus(SStreamTask* pTask) {
+void streamTaskCheckDownstream(SStreamTask* pTask) {
   SDataRange*  pRange = &pTask->dataRange;
   STimeWindow* pWindow = &pRange->window;
 
@@ -152,6 +126,8 @@ static int32_t doCheckDownstreamStatus(SStreamTask* pTask) {
       .stage = pTask->pMeta->stage,
   };
 
+  ASSERT(pTask->status.downstreamReady == 0);
+
   // serialize streamProcessScanHistoryFinishRsp
   if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
     req.reqId = tGenIdPI64();
@@ -187,14 +163,8 @@ static int32_t doCheckDownstreamStatus(SStreamTask* pTask) {
     }
   } else {
     stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", pTask->id.idStr, pTask->info.nodeId);
-
+    doProcessDownstreamReadyRsp(pTask);
-    streamTaskSetReady(pTask, 0);
-    streamTaskSetRangeStreamCalc(pTask);
-    streamTaskStartScanHistory(pTask);
-    streamLaunchFillHistoryTask(pTask);
   }
-
-  return 0;
 }
 
 static STaskRecheckInfo* createRecheckInfo(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
@@ -280,56 +250,88 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
   }
 
   if (pInfo->stage != stage) {
-    return TASK_SELF_NEW_STAGE;
+    return TASK_UPSTREAM_NEW_STAGE;
   } else if (pTask->status.downstreamReady != 1) {
+    stDebug("s-task:%s vgId:%d leader:%d, downstream not ready", id, vgId, (pTask->pMeta->role == NODE_ROLE_LEADER));
     return TASK_DOWNSTREAM_NOT_READY;
   } else {
     return TASK_DOWNSTREAM_READY;
   }
 }
 
-static void doProcessDownstreamReadyRsp(SStreamTask* pTask, int32_t numOfReqs) {
+int32_t onNormalTaskReady(SStreamTask* pTask) {
-  streamTaskSetReady(pTask, numOfReqs);
   const char* id = pTask->id.idStr;
 
-  int8_t      status = pTask->status.taskStatus;
+  streamTaskSetReady(pTask);
-  const char* str = streamGetTaskStatusStr(status);
-
-  ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__NORMAL);
   streamTaskSetRangeStreamCalc(pTask);
 
-  if (status == TASK_STATUS__SCAN_HISTORY) {
+  char* p = NULL;
-    stDebug("s-task:%s enter into scan-history data stage, status:%s", id, str);
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
-    streamTaskStartScanHistory(pTask);
+  ASSERT(status == TASK_STATUS__READY);
-  } else {
+
-    // fill-history tasks are not allowed to reach here.
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    if (pTask->info.fillHistory == 1) {
+    int64_t startVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
-      stDebug("s-task:%s fill-history is set normal when start it, try to remove it,set it task to be dropping", id);
+    if (startVer == -1) {
-      pTask->status.taskStatus = TASK_STATUS__DROPPING;
+      startVer = pTask->chkInfo.nextProcessVer;
-      ASSERT(pTask->hTaskInfo.id.taskId == 0);
-    } else {
-      stDebug("s-task:%s downstream tasks are ready, now ready for data from wal, status:%s", id, str);
-      streamTaskEnablePause(pTask);
     }
+
+    stDebug("s-task:%s no need to scan-history data, status:%s, sched-status:%d, ready for data from wal ver:%" PRId64,
+            id, p, pTask->status.schedStatus, startVer);
+  } else {
+    stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p, pTask->status.schedStatus);
   }
 
-  // when current stream task is ready, check the related fill history task.
+  streamTaskEnablePause(pTask);
-  streamLaunchFillHistoryTask(pTask);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t onScanhistoryTaskReady(SStreamTask* pTask) {
+  const char* id = pTask->id.idStr;
+
+  // set the state to be ready
+  streamTaskSetReady(pTask);
+  streamTaskSetRangeStreamCalc(pTask);
+
+  char* p = NULL;
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
+  ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY);
+
+  stDebug("s-task:%s enter into scan-history data stage, status:%s", id, p);
+  streamTaskStartScanHistory(pTask);
+
+  // start the related fill-history task, when current task is ready
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
+    streamLaunchFillHistoryTask(pTask);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void doProcessDownstreamReadyRsp(SStreamTask* pTask) {
+  EStreamTaskEvent event;
+  if (pTask->info.fillHistory == 0) {
+    event = HAS_RELATED_FILLHISTORY_TASK(pTask)? TASK_EVENT_INIT_STREAM_SCANHIST:TASK_EVENT_INIT;
+  } else {
+    event = TASK_EVENT_INIT_SCANHIST;
+  }
+
+  streamTaskOnHandleEventSuccess(pTask->status.pSM, event);
 }
 
 int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
   ASSERT(pTask->id.taskId == pRsp->upstreamTaskId);
   const char* id = pTask->id.idStr;
+  int32_t vgId = pTask->pMeta->vgId;
 
-  if (streamTaskShouldStop(&pTask->status)) {
+  if (streamTaskShouldStop(pTask)) {
     stDebug("s-task:%s should stop, do not do check downstream again", id);
     return TSDB_CODE_SUCCESS;
   }
 
   if (pRsp->status == TASK_DOWNSTREAM_READY) {
     if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
-      bool found = false;
 
+      bool found = false;
       int32_t numOfReqs = taosArrayGetSize(pTask->checkReqIds);
       for (int32_t i = 0; i < numOfReqs; i++) {
         int64_t reqId = *(int64_t*)taosArrayGet(pTask->checkReqIds, i);
@@ -350,7 +352,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs
         taosArrayDestroy(pTask->checkReqIds);
         pTask->checkReqIds = NULL;
 
-        doProcessDownstreamReadyRsp(pTask, numOfReqs);
+        doProcessDownstreamReadyRsp(pTask);
       } else {
         int32_t total = taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos);
         stDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id,
@@ -362,27 +364,49 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs
         return -1;
       }
 
-      doProcessDownstreamReadyRsp(pTask, 1);
+      doProcessDownstreamReadyRsp(pTask);
     }
   } else {  // not ready, wait for 100ms and retry
-    if (pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) {
+    if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) {
       stError(
-          "s-task:%s downstream taskId:0x%x (vgId:%d) vnode-transfer/leader-change detected, not send check again, "
+          "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, "
-          "roll-back needed",
+          "not check wait for downstream task nodeUpdate, and all tasks restart",
-          id, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
+          id, pRsp->upstreamNodeId, pRsp->oldStage, (int32_t)pTask->pMeta->stage);
-    } else if (pRsp->status == TASK_SELF_NEW_STAGE) {
-      stError(
-          "s-task:%s vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, not send check "
-          "again, roll-back needed",
-          id, pRsp->oldStage, (int32_t)pTask->pMeta->stage);
     } else {
+      if (pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) {
+        stError(
+            "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check "
+            "downstream again, nodeUpdate needed",
+            id, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
+
+        taosThreadMutexLock(&pTask->lock);
+        int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList);
+        bool existed = false;
+        for (int i = 0; i < num; ++i) {
+          SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, i);
+          if (p->nodeId == pRsp->downstreamNodeId) {
+            existed = true;
+            break;
+          }
+        }
+
+        if (!existed) {
+          SDownstreamTaskEpset t = {.nodeId = pRsp->downstreamNodeId};
+          taosArrayPush(pTask->outputInfo.pDownstreamUpdateList, &t);
+          stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", id, vgId,
+                 t.nodeId, (int32_t)taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList));
+        }
+
+        taosThreadMutexUnlock(&pTask->lock);
+        return 0;
+      }
+
       STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp);
 
       int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
       stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%d, retry in 100ms, ref:%d ", id,
               pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->oldStage, ref);
-
+      pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer);
-      taosTmrReset(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer, &pInfo->checkTimer);
     }
   }
 
@@ -426,30 +450,6 @@ int32_t streamRestoreParam(SStreamTask* pTask) {
   return qRestoreStreamOperatorOption(pTask->exec.pExecutor);
 }
 
-int32_t streamSetStatusNormal(SStreamTask* pTask) {
-  int32_t status = atomic_load_8(&pTask->status.taskStatus);
-  if (status == TASK_STATUS__DROPPING) {
-    stError("s-task:%s cannot be set normal, since in dropping state", pTask->id.idStr);
-    return -1;
-  } else {
-    stDebug("s-task:%s set task status to be normal, prev:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
-    return 0;
-  }
-}
-
-int32_t streamSetStatusUnint(SStreamTask* pTask) {
-  int32_t status = atomic_load_8(&pTask->status.taskStatus);
-  if (status == TASK_STATUS__DROPPING) {
-    stError("s-task:%s cannot be set uninit, since in dropping state", pTask->id.idStr);
-    return -1;
-  } else {
-    stDebug("s-task:%s set task status to be uninit, prev:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__UNINIT);
-    return 0;
-  }
-}
-
 // source
 int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange *pVerRange, STimeWindow* pWindow) {
   return qStreamSourceScanParamForHistoryScanStep1(pTask->exec.pExecutor, pVerRange, pWindow);
@@ -515,9 +515,13 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory
   int32_t taskLevel = pTask->info.taskLevel;
   ASSERT(taskLevel == TASK_LEVEL__AGG || taskLevel == TASK_LEVEL__SINK);
 
-  if (pTask->status.taskStatus != TASK_STATUS__SCAN_HISTORY) {
+  const char* id = pTask->id.idStr;
+  char*       p = NULL;
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
+
+  if (status != TASK_STATUS__SCAN_HISTORY && status != TASK_STATUS__STREAM_SCAN_HISTORY) {
     stError("s-task:%s not in scan-history status, status:%s return upstream:0x%x scan-history finish directly",
-           pTask->id.idStr, streamGetTaskStatusStr(pTask->status.taskStatus), pReq->upstreamTaskId);
+           id, p, pReq->upstreamTaskId);
 
     void*   pBuf = NULL;
     int32_t len = 0;
@@ -527,8 +531,8 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory
     initRpcMsg(&msg, 0, pBuf, sizeof(SMsgHead) + len);
 
     tmsgSendRsp(&msg);
-    stDebug("s-task:%s level:%d notify upstream:0x%x(vgId:%d) to continue process data in WAL", pTask->id.idStr,
+    stDebug("s-task:%s level:%d notify upstream:0x%x(vgId:%d) to continue process data in WAL", id,
-           pTask->info.taskLevel, pReq->upstreamTaskId, pReq->upstreamNodeId);
+            taskLevel, pReq->upstreamTaskId, pReq->upstreamNodeId);
     return 0;
   }
 
@@ -540,13 +544,15 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory
 
   if (left == 0) {
     int32_t numOfTasks = taosArrayGetSize(pTask->upstreamInfo.pList);
-    stDebug(
+    if (taskLevel == TASK_LEVEL__AGG) {
-        "s-task:%s all %d upstream tasks finish scan-history data, set param for agg task for stream data and send "
+      stDebug(
-        "rsp to all upstream tasks",
+          "s-task:%s all %d upstream tasks finish scan-history data, set param for agg task for stream data processing "
-        pTask->id.idStr, numOfTasks);
+          "and send rsp to all upstream tasks",
-
+          id, numOfTasks);
-    if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
       streamAggUpstreamScanHistoryFinish(pTask);
+    } else {
+      stDebug("s-task:%s all %d upstream task(s) finish scan-history data, and rsp to all upstream tasks", id,
+              numOfTasks);
     }
 
     // all upstream tasks have completed the scan-history task in the stream time window, let's start to extract data
@@ -557,26 +563,25 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory
     if (taskLevel == TASK_LEVEL__AGG) {
       /*int32_t code = */streamTaskScanHistoryDataComplete(pTask);
     } else {  // for sink task, set normal
-      if (pTask->status.taskStatus != TASK_STATUS__PAUSE && pTask->status.taskStatus != TASK_STATUS__STOP &&
+      streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
-          pTask->status.taskStatus != TASK_STATUS__DROPPING) {
-        streamSetStatusNormal(pTask);
-      }
     }
   } else {
     stDebug("s-task:%s receive scan-history data finish msg from upstream:0x%x(index:%d), unfinished:%d",
-           pTask->id.idStr, pReq->upstreamTaskId, pReq->childId, left);
+           id, pReq->upstreamTaskId, pReq->childId, left);
   }
 
   return 0;
 }
 
 int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) {
-  ASSERT(pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY);
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
+
+  ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY);
   SStreamMeta* pMeta = pTask->pMeta;
 
   // execute in the scan history complete call back msg, ready to process data from inputQ
-  streamSetStatusNormal(pTask);
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
-  streamTaskSetSchedStatusInActive(pTask);
+  streamTaskSetSchedStatusInactive(pTask);
 
   taosWLockLatch(&pMeta->lock);
   streamMetaSaveTask(pMeta, pTask);
@@ -604,15 +609,15 @@ static void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask)
 
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     stDebug("s-task:%s set the launch condition for fill-history s-task:%s, window:%" PRId64 " - %" PRId64
-           " ver range:%" PRId64 " - %" PRId64", init:%"PRId64,
+           " verRange:%" PRId64 " - %" PRId64", init:%"PRId64,
            pTask->id.idStr, pHTask->id.idStr, pRange->window.skey, pRange->window.ekey,
            pRange->range.minVer, pRange->range.maxVer, pHTask->execInfo.init);
   } else {
-    stDebug("s-task:%s no fill history condition for non-source task:%s", pTask->id.idStr, pHTask->id.idStr);
+    stDebug("s-task:%s no fill-history condition for non-source task:%s", pTask->id.idStr, pHTask->id.idStr);
   }
 
   // check if downstream tasks have been ready
-  doCheckDownstreamStatus(pHTask);
+  streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT_SCANHIST);
 }
 
 static void tryLaunchHistoryTask(void* param, void* tmrId) {
@@ -624,12 +629,12 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) {
   if (ppTask) {
     ASSERT((*ppTask)->status.timerActive >= 1);
 
-    if (streamTaskShouldStop(&(*ppTask)->status)) {
+    if (streamTaskShouldStop(*ppTask)) {
-      const char* pStatus = streamGetTaskStatusStr((*ppTask)->status.taskStatus);
+      char* p = NULL;
-
+      streamTaskGetStatus((*ppTask), &p);
       int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1);
       stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d",
-              (*ppTask)->id.idStr, pStatus, (*ppTask)->hTaskInfo.retryTimes, ref);
+              (*ppTask)->id.idStr, p, (*ppTask)->hTaskInfo.retryTimes, ref);
 
       taosMemoryFree(pInfo);
       taosWUnLockLatch(&pMeta->lock);
@@ -665,9 +670,11 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) {
 
       // abort the timer if intend to stop task
       SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId);
-      if (pHTask == NULL && (!streamTaskShouldStop(&pTask->status))) {
+      if (pHTask == NULL && (!streamTaskShouldStop(pTask))) {
-        const char* p = streamGetTaskStatusStr(pTask->status.taskStatus);
+        char*   p = NULL;
-        int32_t     hTaskId = pHTaskInfo->id.taskId;
+        int32_t hTaskId = pHTaskInfo->id.taskId;
+
+        streamTaskGetStatus(pTask, &p);
         stDebug(
             "s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d",
             pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes);
@@ -713,11 +720,8 @@ SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, int64_t streamId, in
 int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
   SStreamMeta* pMeta = pTask->pMeta;
   int32_t      hTaskId = pTask->hTaskInfo.id.taskId;
-  if (hTaskId == 0) {
-    return TSDB_CODE_SUCCESS;
-  }
 
-  ASSERT(pTask->status.downstreamReady == 1);
+  ASSERT((hTaskId != 0) && (pTask->status.downstreamReady == 1));
   stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", pTask->id.idStr,
           pTask->hTaskInfo.id.streamId, hTaskId);
 
@@ -765,7 +769,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
 }
 
 int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask) {
-  if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__DROPPING) {
     return 0;
   }
 
@@ -897,7 +901,7 @@ int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistory
 void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
   SDataRange* pRange = &pTask->dataRange;
 
-  if (pTask->hTaskInfo.id.taskId == 0) {
+  if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     if (pTask->info.fillHistory == 1) {
       stDebug("s-task:%s fill-history task, time window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 "-%" PRId64,
               pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer);
@@ -929,21 +933,8 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
   }
 }
 
-// only the downstream tasks are ready, set the task to be ready to work.
-void streamTaskCheckDownstream(SStreamTask* pTask) {
-  if (pTask->info.fillHistory) {
-    stDebug("s-task:%s fill history task, wait for being launched", pTask->id.idStr);
-    return;
-  }
-
-  ASSERT(pTask->status.downstreamReady == 0);
-  doCheckDownstreamStatus(pTask);
-}
-
-// normal -> pause, pause/stop/dropping -> pause, halt -> pause, scan-history -> pause
 void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta) {
-  int64_t st = taosGetTimestampMs();
+#if 0
-
   int8_t status = pTask->status.taskStatus;
   if (status == TASK_STATUS__DROPPING) {
     stDebug("vgId:%d s-task:%s task already dropped, do nothing", pMeta->vgId, pTask->id.idStr);
@@ -1001,43 +992,54 @@ void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta) {
   stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
   taosWUnLockLatch(&pMeta->lock);
 
+#endif
+
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_PAUSE);
+
+  int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
+  stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
+
   // in case of fill-history task, stop the tsdb file scan operation.
   if (pTask->info.fillHistory == 1) {
     void* pExecutor = pTask->exec.pExecutor;
     qKillTask(pExecutor, TSDB_CODE_SUCCESS);
   }
 
-  int64_t el = taosGetTimestampMs() - st;
+  stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr);
-  stDebug("vgId:%d s-task:%s set pause flag, prev:%s, pause elapsed time:%dms", pMeta->vgId, pTask->id.idStr,
-         streamGetTaskStatusStr(pTask->status.keepTaskStatus), (int32_t)el);
 }
 
-void streamTaskResume(SStreamTask* pTask, SStreamMeta* pMeta) {
+void streamTaskResume(SStreamTask* pTask) {
-  int8_t status = pTask->status.taskStatus;
+  char*        p = NULL;
-  if (status == TASK_STATUS__PAUSE) {
+  ETaskStatus  status = streamTaskGetStatus(pTask, &p);
-    pTask->status.taskStatus = pTask->status.keepTaskStatus;
+  SStreamMeta* pMeta = pTask->pMeta;
-    pTask->status.keepTaskStatus = TASK_STATUS__NORMAL;
+
-    int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
+  if (status == TASK_STATUS__PAUSE || status == TASK_STATUS__HALT) {
-    stInfo("vgId:%d s-task:%s resume from pause, status:%s. pause task num:%d", pMeta->vgId, pTask->id.idStr, streamGetTaskStatusStr(status), num);
+    streamTaskRestoreStatus(pTask);
-  } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
+
-    int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
+    char* pNew = NULL;
-    stInfo("vgId:%d s-task:%s sink task.resume from pause, status:%s. pause task num:%d", pMeta->vgId, pTask->id.idStr, streamGetTaskStatusStr(status), num);
+    streamTaskGetStatus(pTask, &pNew);
+    if (status == TASK_STATUS__PAUSE) {
+      int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
+      stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, p, num);
+    } else {
+      stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, p);
+    }
   } else {
-    stError("s-task:%s not in pause, failed to resume, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
+    stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, p);
   }
 }
 
 // todo fix race condition
 void streamTaskDisablePause(SStreamTask* pTask) {
   // pre-condition check
-  const char* id = pTask->id.idStr;
+//  const char* id = pTask->id.idStr;
-  while (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+//  while (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-    stDebug("s-task:%s already in pause, wait for pause being cancelled, and set pause disabled, recheck in 100ms", id);
+//    stDebug("s-task:%s already in pause, wait for pause being cancelled, and set pause disabled, recheck in 100ms", id);
-    taosMsleep(100);
+//    taosMsleep(100);
-  }
+//  }
-
+//
-  stDebug("s-task:%s disable task pause", id);
+//  stDebug("s-task:%s disable task pause", id);
-  pTask->status.pauseAllowed = 0;
+//  pTask->status.pauseAllowed = 0;
 }
 
 void streamTaskEnablePause(SStreamTask* pTask) {
@@ -1045,15 +1047,30 @@ void streamTaskEnablePause(SStreamTask* pTask) {
   pTask->status.pauseAllowed = 1;
 }
 
-void streamTaskResumeFromHalt(SStreamTask* pTask) {
+int32_t streamMetaUpdateTaskReadyInfo(SStreamTask* pTask) {
-  const char* id = pTask->id.idStr;
+  SStreamMeta* pMeta = pTask->pMeta;
-  int8_t status = pTask->status.taskStatus;
+
-  if (status != TASK_STATUS__HALT) {
+  taosWLockLatch(&pMeta->lock);
-    stError("s-task:%s not in halt status, status:%s", id, streamGetTaskStatusStr(status));
+
-    return;
+  STaskId id = streamTaskExtractKey(pTask);
+  taosHashPut(pMeta->startInfo.pReadyTaskSet, &id, sizeof(id), NULL, 0);
+
+  int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta);
+
+  if (taosHashGetSize(pMeta->startInfo.pReadyTaskSet) == numOfTotal) {
+    STaskStartInfo* pStartInfo = &pMeta->startInfo;
+
+    pStartInfo->readyTs = pTask->execInfo.start;
+    pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0;
+
+    streamMetaResetStartInfo(pStartInfo);
+
+    stDebug("vgId:%d all %d task(s) are started successfully, last ready task:%s level:%d, startTs:%" PRId64
+            ", readyTs:%" PRId64 " total elapsed time:%.2fs",
+            pMeta->vgId, numOfTotal, pTask->id.idStr, pTask->info.taskLevel, pStartInfo->startTs, pStartInfo->readyTs,
+            pStartInfo->elapsedTime / 1000.0);
   }
 
-  pTask->status.taskStatus = pTask->status.keepTaskStatus;
+  taosWUnLockLatch(&pMeta->lock);
-  pTask->status.keepTaskStatus = TASK_STATUS__NORMAL;
+  return TSDB_CODE_SUCCESS;
-  stDebug("s-task:%s resume from halt, current status:%s", id, streamGetTaskStatusStr(pTask->status.taskStatus));
 }

source/libs/stream/src/streamTask.c

@@ -19,6 +19,7 @@
 #include "tstream.h"
 #include "ttimer.h"
 #include "wal.h"
+#include "streamsm.h"
 
 static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo);
 
@@ -34,8 +35,11 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, bool fillHistory
   SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
   if (pTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
+    stError("s-task:0x%" PRIx64 " failed malloc new stream task, size:%d, code:%s", streamId,
+            (int32_t)sizeof(SStreamTask), tstrerror(terrno));
     return NULL;
   }
+
   pTask->ver = SSTREAM_TASK_VER;
   pTask->id.taskId = tGenIdPI32();
   pTask->id.streamId = streamId;
@@ -43,13 +47,19 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, bool fillHistory
   pTask->info.fillHistory = fillHistory;
   pTask->info.triggerParam = triggerParam;
 
+  pTask->status.pSM = streamCreateStateMachine(pTask);
+  if (pTask->status.pSM == NULL) {
+    taosMemoryFreeClear(pTask);
+    return NULL;
+  }
+
   char buf[128] = {0};
   sprintf(buf, "0x%" PRIx64 "-%d", pTask->id.streamId, pTask->id.taskId);
 
   pTask->id.idStr = taosStrdup(buf);
   pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
-  pTask->status.taskStatus = (fillHistory || hasFillhistory)? TASK_STATUS__SCAN_HISTORY:TASK_STATUS__NORMAL;
+  pTask->status.taskStatus = (fillHistory || hasFillhistory) ? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY;
-  pTask->inputInfo.status = TASK_INPUT_STATUS__NORMAL;
+  pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
   pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
 
   if (fillHistory) {
@@ -296,9 +306,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
   int32_t taskId = pTask->id.taskId;
 
   STaskExecStatisInfo* pStatis = &pTask->execInfo;
-
+  stDebug("start to free s-task:0x%x, %p, state:%p", taskId, pTask, pTask->pState);
-  stDebug("start to free s-task:0x%x, %p, state:%p, status:%s", taskId, pTask, pTask->pState,
-         streamGetTaskStatusStr(pTask->status.taskStatus));
 
   stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64
          ", updateCount:%d latestUpdate:%" PRId64 ", latestCheckPoint:%" PRId64 ", ver:%" PRId64
@@ -309,8 +317,8 @@ void tFreeStreamTask(SStreamTask* pTask) {
 
   // remove the ref by timer
   while (pTask->status.timerActive > 0) {
-    stDebug("s-task:%s wait for task stop timer activities", pTask->id.idStr);
+    stDebug("s-task:%s wait for task stop timer activities, ref:%d", pTask->id.idStr, pTask->status.timerActive);
-    taosMsleep(10);
+    taosMsleep(100);
   }
 
   if (pTask->schedInfo.pTimer != NULL) {
@@ -329,8 +337,8 @@ void tFreeStreamTask(SStreamTask* pTask) {
   }
 
   int32_t status = atomic_load_8((int8_t*)&(pTask->status.taskStatus));
-  if (pTask->inputInfo.queue) {
+  if (pTask->inputq.queue) {
-    streamQueueClose(pTask->inputInfo.queue, pTask->id.taskId);
+    streamQueueClose(pTask->inputq.queue, pTask->id.taskId);
   }
 
   if (pTask->outputq.queue) {
@@ -384,13 +392,18 @@ void tFreeStreamTask(SStreamTask* pTask) {
     pTask->pRspMsgList = NULL;
   }
 
+  pTask->status.pSM = streamDestroyStateMachine(pTask->status.pSM);
+
   streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo);
 
   pTask->msgInfo.pRetryList = taosArrayDestroy(pTask->msgInfo.pRetryList);
   taosMemoryFree(pTask->outputInfo.pTokenBucket);
   taosThreadMutexDestroy(&pTask->lock);
-  taosMemoryFree(pTask);
 
+  taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList);
+  pTask->outputInfo.pDownstreamUpdateList = NULL;
+
+  taosMemoryFree(pTask);
   stDebug("s-task:0x%x free task completed", taskId);
 }
 
@@ -399,16 +412,23 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
   pTask->refCnt = 1;
   pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
   pTask->status.timerActive = 0;
-  pTask->inputInfo.queue = streamQueueOpen(512 << 10);
+  pTask->inputq.queue = streamQueueOpen(512 << 10);
   pTask->outputq.queue = streamQueueOpen(512 << 10);
 
-  if (pTask->inputInfo.queue == NULL || pTask->outputq.queue == NULL) {
+  if (pTask->inputq.queue == NULL || pTask->outputq.queue == NULL) {
     stError("s-task:%s failed to prepare the input/output queue, initialize task failed", pTask->id.idStr);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pTask->status.pSM = streamCreateStateMachine(pTask);
+  if (pTask->status.pSM == NULL) {
+    stError("s-task:%s failed create state-machine for stream task, initialization failed, code:%s", pTask->id.idStr,
+        tstrerror(terrno));
+    return terrno;
+  }
+
   pTask->execInfo.created = taosGetTimestampMs();
-  pTask->inputInfo.status = TASK_INPUT_STATUS__NORMAL;
+  pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
   pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
   pTask->pMeta = pMeta;
 
@@ -427,7 +447,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
 
   // 2MiB per second for sink task
   // 50 times sink operator per second
-  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, 2);
+  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate);
 
   TdThreadMutexAttr attr = {0};
   int code = taosThreadMutexAttrInit(&attr);
@@ -445,6 +465,11 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
   taosThreadMutexInit(&pTask->lock, &attr);
   streamTaskOpenAllUpstreamInput(pTask);
 
+  pTask->outputInfo.pDownstreamUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset));
+  if (pTask->outputInfo.pDownstreamUpdateList == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -453,7 +478,9 @@ int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) {
     return 0;
   } else {
     int32_t type = pTask->outputInfo.type;
-    if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__TABLE) {
+    if (type == TASK_OUTPUT__TABLE) {
+      return 0;
+    } else if (type == TASK_OUTPUT__FIXED_DISPATCH) {
       return 1;
     } else {
       SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
@@ -562,13 +589,7 @@ int32_t streamTaskStop(SStreamTask* pTask) {
   int64_t      st = taosGetTimestampMs();
   const char*  id = pTask->id.idStr;
 
-  taosThreadMutexLock(&pTask->lock);
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP);
-  if (pTask->status.taskStatus == TASK_STATUS__CK) {
-    stDebug("s-task:%s in checkpoint will be discarded since task is stopped", id);
-  }
-  pTask->status.taskStatus = TASK_STATUS__STOP;
-  taosThreadMutexUnlock(&pTask->lock);
-
   qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
   while (/*pTask->status.schedStatus != TASK_SCHED_STATUS__INACTIVE */ !streamTaskIsIdle(pTask)) {
     stDebug("s-task:%s level:%d wait for task to be idle and then close, check again in 100ms", id,
@@ -612,7 +633,7 @@ int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) {
 
   p->latestUpdateTs = taosGetTimestampMs();
   p->updateCount += 1;
-  stDebug("s-task:%s update task nodeEp epset, updatedNodes:%d, updateCount:%d, prevTs:%" PRId64, pTask->id.idStr,
+  stDebug("s-task:0x%x update task nodeEp epset, updatedNodes:%d, updateCount:%d, prevTs:%" PRId64, pTask->id.taskId,
          numOfNodes, p->updateCount, prevTs);
 
   for (int32_t i = 0; i < taosArrayGetSize(pNodeList); ++i) {
@@ -667,7 +688,7 @@ int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) {
   return status;
 }
 
-int8_t streamTaskSetSchedStatusInActive(SStreamTask* pTask) {
+int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
   taosThreadMutexLock(&pTask->lock);
   int8_t status = pTask->status.schedStatus;
   ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE ||
@@ -678,6 +699,25 @@ int8_t streamTaskSetSchedStatusInActive(SStreamTask* pTask) {
   return status;
 }
 
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask) {
+  SStreamMeta* pMeta = pTask->pMeta;
+  if (pTask->info.fillHistory == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId};
+  SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId));
+
+  if (ppStreamTask != NULL) {
+    CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask));
+    streamMetaSaveTask(pMeta, *ppStreamTask);
+    stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr,
+            (int32_t)sTaskId.taskId);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId) {
   SVDropStreamTaskReq *pReq = rpcMallocCont(sizeof(SVDropStreamTaskReq));
   if (pReq == NULL) {
@@ -696,7 +736,7 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
     return code;
   }
 
-  stDebug("vgId:%d build and send drop table:0x%x msg", vgId, pTaskId->taskId);
+  stDebug("vgId:%d build and send drop task:0x%x msg", vgId, pTaskId->taskId);
   return code;
 }
 
@@ -719,20 +759,6 @@ void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo) {
   pInfo->retryTimes += 1;
 }
 
-const char* streamGetTaskStatusStr(int32_t status) {
-  switch(status) {
-    case TASK_STATUS__NORMAL: return "normal";
-    case TASK_STATUS__SCAN_HISTORY: return "scan-history";
-    case TASK_STATUS__HALT: return "halt";
-    case TASK_STATUS__PAUSE: return "paused";
-    case TASK_STATUS__CK: return "check-point";
-    case TASK_STATUS__DROPPING: return "dropping";
-    case TASK_STATUS__STOP: return "stop";
-    case TASK_STATUS__UNINIT: return "uninitialized";
-    default:return "";
-  }
-}
-
 void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask) {
   pEntry->id.streamId = pTask->id.streamId;
   pEntry->id.taskId = pTask->id.taskId;

source/libs/stream/src/streamTaskSm.c

@@ -0,0 +1,529 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "streamInt.h"
+#include "streamsm.h"
+#include "tmisce.h"
+#include "tstream.h"
+#include "ttimer.h"
+#include "wal.h"
+
+SStreamTaskState StreamTaskStatusList[9] = {
+    {.state = TASK_STATUS__READY, .name = "ready"},
+    {.state = TASK_STATUS__DROPPING, .name = "dropped"},
+    {.state = TASK_STATUS__UNINIT, .name = "uninit"},
+    {.state = TASK_STATUS__STOP, .name = "stop"},
+    {.state = TASK_STATUS__SCAN_HISTORY, .name = "scan-history"},
+    {.state = TASK_STATUS__HALT, .name = "halt"},
+    {.state = TASK_STATUS__PAUSE, .name = "paused"},
+    {.state = TASK_STATUS__CK, .name = "checkpoint"},
+    {.state = TASK_STATUS__STREAM_SCAN_HISTORY, .name = "stream-scan-history"},
+};
+
+SStreamEventInfo StreamTaskEventList[12] = {
+    {.event = 0, .name = ""},  // dummy event, place holder
+    {.event = TASK_EVENT_INIT, .name = "initialize"},
+    {.event = TASK_EVENT_INIT_SCANHIST, .name = "scan-history-init"},
+    {.event = TASK_EVENT_INIT_STREAM_SCANHIST, .name = "stream-scan-history-init"},
+    {.event = TASK_EVENT_SCANHIST_DONE, .name = "scan-history-completed"},
+    {.event = TASK_EVENT_STOP, .name = "stopping"},
+    {.event = TASK_EVENT_GEN_CHECKPOINT, .name = "checkpoint"},
+    {.event = TASK_EVENT_CHECKPOINT_DONE, .name = "checkpoint-done"},
+    {.event = TASK_EVENT_PAUSE, .name = "pausing"},
+    {.event = TASK_EVENT_RESUME, .name = "resuming"},
+    {.event = TASK_EVENT_HALT, .name = "halting"},
+    {.event = TASK_EVENT_DROPPING, .name = "dropping"},
+};
+
+static TdThreadOnce streamTaskStateMachineInit = PTHREAD_ONCE_INIT;
+static SArray*      streamTaskSMTrans = NULL;
+
+static int32_t streamTaskInitStatus(SStreamTask* pTask);
+static int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask);
+static int32_t initStateTransferTable();
+static void    doInitStateTransferTable(void);
+
+static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event,
+                                            __state_trans_fn fn, __state_trans_succ_fn succFn,
+                                            SAttachedEventInfo* pEventInfo, bool autoInvoke);
+
+static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; }
+
+static int32_t attachEvent(SStreamTask* pTask, SAttachedEventInfo* pEvtInfo) {
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
+  stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p,
+          StreamTaskEventList[pEvtInfo->event].name, StreamTaskStatusList[pEvtInfo->status].name);
+  taosArrayPush(pTask->status.pSM->pWaitingEventList, pEvtInfo);
+  return 0;
+}
+
+int32_t streamTaskInitStatus(SStreamTask* pTask) {
+  pTask->execInfo.init = taosGetTimestampMs();
+
+  stDebug("s-task:%s start init, and check downstream tasks, set the init ts:%" PRId64, pTask->id.idStr,
+          pTask->execInfo.init);
+  streamTaskCheckDownstream(pTask);
+  return 0;
+}
+
+int32_t streamTaskSetReadyForWal(SStreamTask* pTask) {
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    stDebug("s-task:%s ready for extract data from wal", pTask->id.idStr);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t streamTaskDoCheckpoint(SStreamTask* pTask) {
+  stDebug("s-task:%s start to do checkpoint", pTask->id.idStr);
+  return 0;
+}
+
+int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) {
+  ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));
+
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    pTask->hTaskInfo.haltVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
+    if (pTask->hTaskInfo.haltVer == -1) {
+      pTask->hTaskInfo.haltVer = pTask->dataRange.range.maxVer + 1;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+// todo optimize the perf of find the trans objs by using hash table
+static STaskStateTrans* streamTaskFindTransform(ETaskStatus state, const EStreamTaskEvent event) {
+  int32_t numOfTrans = taosArrayGetSize(streamTaskSMTrans);
+  for (int32_t i = 0; i < numOfTrans; ++i) {
+    STaskStateTrans* pTrans = taosArrayGet(streamTaskSMTrans, i);
+    if (pTrans->state.state == state && pTrans->event == event) {
+      return pTrans;
+    }
+  }
+
+  if (event == TASK_EVENT_CHECKPOINT_DONE && state == TASK_STATUS__STOP) {
+
+  } else if (event == TASK_EVENT_GEN_CHECKPOINT && state == TASK_STATUS__UNINIT) {
+    // the task is set to uninit due to nodeEpset update, during processing checkpoint-trigger block.
+  } else {
+    ASSERT(0);
+  }
+
+  return NULL;
+}
+
+void streamTaskRestoreStatus(SStreamTask* pTask) {
+  SStreamTaskSM* pSM = pTask->status.pSM;
+  taosThreadMutexLock(&pTask->lock);
+  ASSERT(pSM->pActiveTrans == NULL);
+
+  ASSERT(pSM->current.state == TASK_STATUS__PAUSE || pSM->current.state == TASK_STATUS__HALT);
+
+  SStreamTaskState state = pSM->current;
+  pSM->current = pSM->prev.state;
+
+  pSM->prev.state = state;
+  pSM->prev.evt = 0;
+
+  pSM->startTs = taosGetTimestampMs();
+  stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name);
+
+  taosThreadMutexUnlock(&pTask->lock);
+}
+
+SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) {
+  initStateTransferTable();
+  const char* id = pTask->id.idStr;
+
+  SStreamTaskSM* pSM = taosMemoryCalloc(1, sizeof(SStreamTaskSM));
+  if (pSM == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM),
+            tstrerror(terrno));
+    return NULL;
+  }
+
+  pSM->pTask = pTask;
+  pSM->pWaitingEventList = taosArrayInit(4, sizeof(SAttachedEventInfo));
+  if (pSM->pWaitingEventList == NULL) {
+    taosMemoryFree(pSM);
+
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM),
+            tstrerror(terrno));
+    return NULL;
+  }
+
+  // set the initial state for the state-machine of stream task
+  pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
+  pSM->startTs = taosGetTimestampMs();
+  return pSM;
+}
+
+void* streamDestroyStateMachine(SStreamTaskSM* pSM) {
+  if (pSM == NULL) {
+    return NULL;
+  }
+
+  taosArrayDestroy(pSM->pWaitingEventList);
+  taosMemoryFree(pSM);
+  return NULL;
+}
+
+static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans) {
+  SStreamTask* pTask = pSM->pTask;
+  const char*  id = pTask->id.idStr;
+
+  if (pTrans->attachEvent.event != 0) {
+    attachEvent(pTask, &pTrans->attachEvent);
+    taosThreadMutexUnlock(&pTask->lock);
+
+    while (1) {
+      // wait for the task to be here
+      taosThreadMutexLock(&pTask->lock);
+      ETaskStatus s = streamTaskGetStatus(pTask, NULL);
+      taosThreadMutexUnlock(&pTask->lock);
+
+      if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {
+        stDebug("s-task:%s attached event:%s handled", id, StreamTaskEventList[pTrans->event].name);
+        return TSDB_CODE_SUCCESS;
+      } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP) {  // this event has been handled already
+        stDebug("s-task:%s not handle event:%s yet, wait for 100ms and recheck", id, StreamTaskEventList[event].name);
+        taosMsleep(100);
+      } else {
+        stDebug("s-task:%s is dropped or stopped already, not wait.", id);
+        return TSDB_CODE_INVALID_PARA;
+      }
+    }
+
+  } else {  // override current active trans
+    pSM->pActiveTrans = pTrans;
+    pSM->startTs = taosGetTimestampMs();
+    taosThreadMutexUnlock(&pTask->lock);
+
+    int32_t code = pTrans->pAction(pTask);
+    // todo handle error code;
+
+    if (pTrans->autoInvokeEndFn) {
+      streamTaskOnHandleEventSuccess(pSM, event);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
+  SStreamTask*     pTask = pSM->pTask;
+  STaskStateTrans* pTrans = NULL;
+
+  while (1) {
+    taosThreadMutexLock(&pTask->lock);
+    if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
+      taosThreadMutexUnlock(&pTask->lock);
+      taosMsleep(100);
+      stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
+              pTask->id.idStr, pSM->current.name, StreamTaskEventList[pSM->pActiveTrans->event].name);
+    } else {
+      pTrans = streamTaskFindTransform(pSM->current.state, event);
+      if (pTrans == NULL) {
+        stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, StreamTaskEventList[event].name);
+        taosThreadMutexUnlock(&pTask->lock);
+        return TSDB_CODE_INVALID_PARA;  // todo: set new error code// failed to handle the event.
+      }
+
+      if (pSM->pActiveTrans != NULL) {
+        // currently in some state transfer procedure, not auto invoke transfer, abort it
+        stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now",
+                pTask->id.idStr, StreamTaskEventList[pSM->pActiveTrans->event].name, pSM->current.name,
+                pSM->pActiveTrans->next.name, StreamTaskEventList[event].name);
+      }
+
+      doHandleEvent(pSM, event, pTrans);
+      break;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void keepPrevInfo(SStreamTaskSM* pSM) {
+  STaskStateTrans* pTrans = pSM->pActiveTrans;
+
+  pSM->prev.state = pSM->current;
+  pSM->prev.evt = pTrans->event;
+}
+
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event) {
+  SStreamTask* pTask = pSM->pTask;
+
+  // do update the task status
+  taosThreadMutexLock(&pTask->lock);
+  STaskStateTrans* pTrans = pSM->pActiveTrans;
+
+  if (pTrans == NULL) {
+    ETaskStatus s = pSM->current.state;
+    ASSERT(s == TASK_STATUS__DROPPING || s == TASK_STATUS__PAUSE || s == TASK_STATUS__STOP);
+    // the pSM->prev.evt may be 0, so print string is not appropriate.
+    stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", pTask->id.idStr,
+            StreamTaskEventList[event].name, pSM->current.name, StreamTaskEventList[pSM->prev.evt].name);
+
+    taosThreadMutexUnlock(&pTask->lock);
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  if (pTrans->event != event) {
+    stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", pTask->id.idStr,
+           StreamTaskEventList[event].name, pSM->current.name, StreamTaskEventList[pTrans->event].name);
+    taosThreadMutexUnlock(&pTask->lock);
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  keepPrevInfo(pSM);
+
+  pSM->current = pTrans->next;
+  pSM->pActiveTrans = NULL;
+
+  // on success callback, add into lock if necessary, or maybe we should add an option for this?
+  pTrans->pSuccAction(pTask);
+
+  if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
+    int64_t el = (taosGetTimestampMs() - pSM->startTs);
+    stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr,
+            StreamTaskEventList[pTrans->event].name, el, pSM->prev.state.name, pSM->current.name);
+
+    SAttachedEventInfo* pEvtInfo = taosArrayPop(pSM->pWaitingEventList);
+
+    // OK, let's handle the attached event, since the task has reached the required status now
+    if (pSM->current.state == pEvtInfo->status) {
+      stDebug("s-task:%s handle the attached event:%s, state:%s", pTask->id.idStr,
+              StreamTaskEventList[pEvtInfo->event].name, pSM->current.name);
+
+      STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event);
+      ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL);
+
+      pSM->pActiveTrans = pNextTrans;
+      pSM->startTs = taosGetTimestampMs();
+      taosThreadMutexUnlock(&pTask->lock);
+
+      int32_t code = pNextTrans->pAction(pSM->pTask);
+      if (pNextTrans->autoInvokeEndFn) {
+        return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event);
+      } else {
+        return code;
+      }
+    }
+  } else {
+    taosThreadMutexUnlock(&pTask->lock);
+
+    int64_t el = (taosGetTimestampMs() - pSM->startTs);
+    stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr,
+            StreamTaskEventList[pTrans->event].name, el, pSM->prev.state.name, pSM->current.name);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr) {
+  SStreamTaskState s = pTask->status.pSM->current;  // copy one obj in case of multi-thread environment
+  if (pStr != NULL) {
+    *pStr = s.name;
+  }
+  return s.state;
+}
+
+const char* streamTaskGetStatusStr(ETaskStatus status) {
+  return StreamTaskStatusList[status].name;
+}
+
+void streamTaskResetStatus(SStreamTask* pTask) {
+  SStreamTaskSM* pSM = pTask->status.pSM;
+
+  taosThreadMutexLock(&pTask->lock);
+  stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr,
+          pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name);
+
+  pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
+  pSM->pActiveTrans = NULL;
+  taosArrayClear(pSM->pWaitingEventList);
+  taosThreadMutexUnlock(&pTask->lock);
+
+  // clear the downstream ready status
+  pTask->status.downstreamReady = 0;
+}
+
+void streamTaskSetStatusReady(SStreamTask* pTask) {
+  SStreamTaskSM* pSM = pTask->status.pSM;
+  if (pSM->current.state == TASK_STATUS__DROPPING) {
+    stError("s-task:%s task in dropping state, cannot be set ready", pTask->id.idStr);
+    return;
+  }
+
+  taosThreadMutexLock(&pTask->lock);
+
+  pSM->prev.state = pSM->current;
+  pSM->prev.evt = 0;
+
+  pSM->current = StreamTaskStatusList[TASK_STATUS__READY];
+  pSM->startTs = taosGetTimestampMs();
+  pSM->pActiveTrans = NULL;
+  taosArrayClear(pSM->pWaitingEventList);
+
+  taosThreadMutexUnlock(&pTask->lock);
+}
+
+STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn,
+                                     __state_trans_succ_fn succFn, SAttachedEventInfo* pEventInfo, bool autoInvoke) {
+  STaskStateTrans trans = {0};
+  trans.state = StreamTaskStatusList[current];
+  trans.next = StreamTaskStatusList[next];
+  trans.event = event;
+
+  if (pEventInfo != NULL) {
+    trans.attachEvent = *pEventInfo;
+  } else {
+    trans.attachEvent.event = 0;
+    trans.attachEvent.status = 0;
+  }
+
+  trans.pAction = (fn != NULL) ? fn : dummyFn;
+  trans.pSuccAction = (succFn != NULL) ? succFn : dummyFn;
+  trans.autoInvokeEndFn = autoInvoke;
+  return trans;
+}
+
+int32_t initStateTransferTable() {
+  taosThreadOnce(&streamTaskStateMachineInit, doInitStateTransferTable);
+  return TSDB_CODE_SUCCESS;
+}
+
+void doInitStateTransferTable(void) {
+  streamTaskSMTrans = taosArrayInit(8, sizeof(STaskStateTrans));
+
+  // initialization event handle
+  STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT,
+                                               streamTaskInitStatus, onNormalTaskReady, false, false);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST,
+                               streamTaskInitStatus, onScanhistoryTaskReady, false, false);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST,
+                               streamTaskInitStatus, onScanhistoryTaskReady, false, false);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // scan-history related event
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL,
+                               NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL,
+                               NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // halt stream task, from other task status
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL,
+                               streamTaskKeepCurrentVerInWal, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT};
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL,
+                               streamTaskKeepCurrentVerInWal, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal,
+                               &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL,
+                               streamTaskKeepCurrentVerInWal, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // checkpoint related event
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL,
+                               streamTaskDoCheckpoint, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans =
+      createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // pause & resume related event handle
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE};
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // resume is completed by restore status of state-machine
+
+  // stop related event
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // dropping related event
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+}

source/util/src/tarray.c

@@ -191,7 +191,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) {
   }
 
   if (index >= pArray->size) {
-    uError("index is out of range, current:%" PRIzu " max:%d", index, pArray->capacity);
+    uError("index is out of range, current:%" PRIzu " max:%"PRIzu, index, pArray->size);
     return NULL;
   }
 

source/util/src/tconfig.c

@@ -416,16 +416,6 @@ int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float min
   return cfgAddItem(pCfg, &item, name);
 }
 
-int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope) {
-  if (defaultVal < minval || defaultVal > maxval) {
-    terrno = TSDB_CODE_OUT_OF_RANGE;
-    return -1;
-  }
-
-  SConfigItem item = {.dtype = CFG_DTYPE_DOUBLE, .dval = defaultVal, .fmin = minval, .fmax = maxval, .scope = scope};
-  return cfgAddItem(pCfg, &item, name);
-}
-
 int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
   SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope};
   item.str = taosStrdup(defaultVal);
@@ -611,8 +601,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
     switch (pItem->dtype) {
       case CFG_DTYPE_BOOL:
         if (dump) {
-          printf("%s %s %u", src, name, pItem->bval);
+          printf("%s %s %u\n", src, name, pItem->bval);
-          printf("\n");
         } else {
           uInfo("%s %s %u", src, name, pItem->bval);
         }
@@ -620,25 +609,22 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
         break;
       case CFG_DTYPE_INT32:
         if (dump) {
-          printf("%s %s %d", src, name, pItem->i32);
+          printf("%s %s %d\n", src, name, pItem->i32);
-          printf("\n");
         } else {
           uInfo("%s %s %d", src, name, pItem->i32);
         }
         break;
       case CFG_DTYPE_INT64:
         if (dump) {
-          printf("%s %s %" PRId64, src, name, pItem->i64);
+          printf("%s %s %" PRId64"\n", src, name, pItem->i64);
-          printf("\n");
         } else {
           uInfo("%s %s %" PRId64, src, name, pItem->i64);
         }
         break;
-      case CFG_DTYPE_FLOAT:
       case CFG_DTYPE_DOUBLE:
+      case CFG_DTYPE_FLOAT:
         if (dump) {
-          printf("%s %s %.2f", src, name, pItem->fval);
+          printf("%s %s %.2f\n", src, name, pItem->fval);
-          printf("\n");
         } else {
           uInfo("%s %s %.2f", src, name, pItem->fval);
         }
@@ -650,8 +636,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
       case CFG_DTYPE_TIMEZONE:
       case CFG_DTYPE_NONE:
         if (dump) {
-          printf("%s %s %s", src, name, pItem->str);
+          printf("%s %s %s\n", src, name, pItem->str);
-          printf("\n");
         } else {
           uInfo("%s %s %s", src, name, pItem->str);
         }
@@ -660,8 +645,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
   }
 
   if (dump) {
-    printf("=================================================================");
+    printf("=================================================================\n");
-    printf("\n");
   } else {
     uInfo("=================================================================");
   }

source/util/src/terror.c

@@ -259,6 +259,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID,         "Invalid dnode id")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_UN_CHANGED,        "Vgroup distribution has not changed")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_HAS_OFFLINE_DNODE,        "Offline dnode exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_REPLICA,          "Invalid vgroup replica")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_VNODES,         "Vnodes exhausted")
 
 // mnode-stable-part2
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "STable confilct with topic")
@@ -558,7 +559,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TOO_MANY_COLUMNS,           "Too many columns")
 TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FIRST_COLUMN,       "First column must be timestamp")
 TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN,     "Invalid varbinary/binary/nchar column/tag length")
 TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TAGS_NUM,           "Invalid number of tag columns")
-TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PERMISSION_DENIED,          "Permission denied")
+TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PERMISSION_DENIED,          "Permission denied or target object not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STREAM_QUERY,       "Invalid stream query")
 TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_INTERNAL_PK,        "Invalid _c0 or _rowts expression")
 TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TIMELINE_FUNC,      "Invalid timeline function")

source/util/src/tqueue.c

@@ -242,6 +242,11 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) {
     qall->current = queue->head;
     qall->start = queue->head;
     qall->numOfItems = queue->numOfItems;
+    qall->memOfItems = queue->memOfItems;
+
+    qall->unAccessedNumOfItems = queue->numOfItems;
+    qall->unAccessMemOfItems = queue->memOfItems;
+
     numOfItems = qall->numOfItems;
 
     queue->head = NULL;
@@ -274,6 +279,10 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) {
   if (pNode) {
     *ppItem = pNode->item;
     num = 1;
+
+    qall->unAccessedNumOfItems -= 1;
+    qall->unAccessMemOfItems -= pNode->dataSize;
+
     uTrace("item:%p is fetched", *ppItem);
   } else {
     *ppItem = NULL;
@@ -449,6 +458,8 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *
       qall->current = queue->head;
       qall->start = queue->head;
       qall->numOfItems = queue->numOfItems;
+      qall->memOfItems = queue->memOfItems;
+
       code = qall->numOfItems;
       qinfo->ahandle = queue->ahandle;
       qinfo->fp = queue->itemsFp;
@@ -476,6 +487,11 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *
 }
 
 int32_t taosQallItemSize(STaosQall *qall) { return qall->numOfItems; }
+int64_t taosQallMemSize(STaosQall *qall) { return qall->memOfItems; }
+
+int64_t taosQallUnAccessedItemSize(STaosQall *qall) {return qall->unAccessedNumOfItems;}
+int64_t taosQallUnAccessedMemSize(STaosQall *qall) {return qall->unAccessMemOfItems;}
+
 void    taosResetQitems(STaosQall *qall) { qall->current = qall->start; }
 int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; }
 

tests/pytest/util/dnodes.py

@@ -130,14 +130,16 @@ class TDDnode:
             "locale": "en_US.UTF-8",
             "charset": "UTF-8",
             "asyncLog": "0",
-            "mDebugFlag": "135",
+            "mDebugFlag": "143",
-            "dDebugFlag": "135",
+            "dDebugFlag": "143",
-            "vDebugFlag": "135",
+            "vDebugFlag": "143",
-            "tqDebugFlag": "135",
+            "tqDebugFlag": "143",
-            "cDebugFlag": "135",
+            "cDebugFlag": "143",
-            "jniDebugFlag": "135",
+            "stDebugFlag": "143",
-            "qDebugFlag": "135",
+            "smaDebugFlag": "143",
-            "rpcDebugFlag": "135",
+            "jniDebugFlag": "143",
+            "qDebugFlag": "143",
+            "rpcDebugFlag": "143",
             "tmrDebugFlag": "131",
             "uDebugFlag": "135",
             "sDebugFlag": "135",

tests/script/tsim/stream/distributeIntervalRetrive0.sim

@@ -267,6 +267,7 @@ sql create table t1 using st tags(1,1,1);
 sql create table t2 using st tags(2,2,2);
 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0  delete_mark 20s into streamt1 as select  _wstart as c0, count(*) c1, count(a) c2  from st interval(10s) ;
 
+sleep 1000
 sql insert into t1 values(1648791211000,1,2,3);
 
 sql insert into t1 values(1262275200000,2,2,3);

tests/script/tsim/stream/pauseAndResume.sim

@@ -16,7 +16,9 @@ sql create table ts2 using st tags(2,2,2);
 sql create table ts3 using st tags(3,2,2);
 sql create table ts4 using st tags(4,2,2);
 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select  _wstart, count(*) c1, sum(a) c3 from st interval(10s);
+sleep 1000
 
+sleep 1000
 sql pause stream streams1;
 
 sql insert into ts1 values(1648791213001,1,12,3,1.0);
@@ -246,6 +248,7 @@ sql create table ts4 using st tags(4,2,2);
 sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt3 as select  _wstart, count(*) c1, sum(a) c3 from st interval(10s);
 sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt4 as select  _wstart, count(*) c1, sum(a) c3 from st interval(10s);
 sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt5 as select  _wstart, count(*) c1, sum(a) c3 from ts1 interval(10s);
+sleep 1000
 
 sql pause stream streams3;
 

tests/system-test/0-others/timeRangeWise.py

@@ -23,7 +23,8 @@ from util.cases import *
 from util.sql import *
 
 class TDTestCase:
-
+    updatecfgDict = {'vdebugFlag': 143, 'qdebugflag':135, 'tqdebugflag':135, 'udebugflag':135, 'rpcdebugflag':135,
+                     'asynclog': 0, 'stdebugflag':135}
     # random string
     def random_string(self, count):
         letters = string.ascii_letters

tests/system-test/8-stream/max_delay_session.py

@@ -35,6 +35,8 @@ class TDTestCase:
         self.tdCom.create_stream(stream_name=f'{self.ctb_name}{self.tdCom.stream_suffix}', des_table=self.ctb_stream_des_table, source_sql=f'select _wstart AS wstart, _wend AS wend, {self.tdCom.stb_source_select_str}  from {self.ctb_name} session(ts, {self.tdCom.dataDict["session"]}s)', trigger_mode="max_delay", watermark=watermark_value, max_delay=max_delay_value, fill_history_value=fill_history_value)
         self.tdCom.create_stream(stream_name=f'{self.tb_name}{self.tdCom.stream_suffix}', des_table=self.tb_stream_des_table, source_sql=f'select _wstart AS wstart, _wend AS wend, {self.tdCom.tb_source_select_str}  from {self.tb_name} session(ts, {self.tdCom.dataDict["session"]}s)', trigger_mode="max_delay", watermark=watermark_value, max_delay=max_delay_value, fill_history_value=fill_history_value)
         init_num = 0
+        time.sleep(1)
+
         for i in range(self.tdCom.range_count):
             if i == 0:
                 window_close_ts = self.tdCom.cal_watermark_window_close_session_endts(self.tdCom.date_time, self.tdCom.dataDict['watermark'], self.tdCom.dataDict['session'])

tests/system-test/8-stream/scalar_function.py

@@ -7,7 +7,7 @@ from util.common import *
 
 class TDTestCase:
     updatecfgDict = {'vdebugFlag': 143, 'qdebugflag':135, 'tqdebugflag':135, 'udebugflag':135, 'rpcdebugflag':135,
-                     'asynclog': 0}
+                     'asynclog': 0, 'stdebugflag':135}
     def init(self, conn, logSql, replicaVar=1):
         self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)

tests/system-test/test.py

@@ -398,7 +398,7 @@ if __name__ == "__main__":
             tdDnodes.setValgrind(valgrind)
             tdDnodes.stopAll()
             for dnode in tdDnodes.dnodes:
-                tdDnodes.deploy(dnode.index,{})
+                tdDnodes.deploy(dnode.index, updateCfgDict)
             for dnode in tdDnodes.dnodes:
                 tdDnodes.starttaosd(dnode.index)
             tdCases.logSql(logSql)