Merge branch '3.0' of github.com:taosdata/TDengine into szhou/tbname-prune-vgroup

2023-11-10 16:45:13 +08:00 · 2023-11-10 16:45:13 +08:00 · 918fadaf00
parent 9d448c7c6c c480f71118
commit 918fadaf00
120 changed files with 5695 additions and 1678 deletions
--- a/2
+++ b/2
@ -316,7 +316,7 @@ def pre_test_build_win() {
        python -m pip uninstall taospy -y 
        python -m pip install taospy==2.7.12
        python -m pip uninstall taos-ws-py -y 
-        python -m pip install taos-ws-py==0.2.9
+        python -m pip install taos-ws-py==0.3.1
 	xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
    '''
    return 1
--- a/cmake/curl_CMakeLists.txt.in
+++ b/cmake/curl_CMakeLists.txt.in
@ -11,7 +11,7 @@ ExternalProject_Add(curl2
        BUILD_IN_SOURCE TRUE
        BUILD_ALWAYS 1
        UPDATE_COMMAND ""
-        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug
+        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 --without-nghttp2 #--enable-debug
        BUILD_COMMAND make -j
        INSTALL_COMMAND make install
        TEST_COMMAND ""
--- a/cmake/ssl_CMakeLists.txt.in
+++ b/cmake/ssl_CMakeLists.txt.in
@ -8,7 +8,7 @@ ExternalProject_Add(openssl
        BUILD_IN_SOURCE TRUE
        #BUILD_ALWAYS 1
        #UPDATE_COMMAND ""
-        CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared
+        CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 no-shared
        BUILD_COMMAND make -j
        INSTALL_COMMAND make install_sw -j
        TEST_COMMAND ""
--- a/deps/x86/rocksdb_static/librocksdb.a
+++ b/deps/x86/rocksdb_static/librocksdb.a
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@ -565,7 +565,7 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal)
 - The uppercase or lowercase of `MONTH`, `MON`, `DAY`, `DY` and formtas that output digits have same effect when used in `to_timestamp`, like `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month` can be replaced by `MONTH`, or `month`. The cases are ignored.
 - If multi times are specified for one component, the previous will be overwritten. Like `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, the output year will be `2022`.
 - To avoid unexpected time zone used during the convertion, it's recommended to put time zone in the ts string, e.g. '2023-10-10 10:10:10+08'. If time zone not specified, default will be that in server or client.
- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with the timezone specified or default to local timezone.
+- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with the timezone specified or default to local timezone. Only `DDD` is specified without `DD` is not supported currently, e.g. format 'yyyy-mm-ddd' is not supported, but 'yyyy-mm-dd' is supported.
 - If `AM` or `PM` is specified in formats, the Hour must between `1-12`.
 - In some cases, `to_timestamp` can convert correctly even the format and the timestamp string are not totally matched. Like `to_timetamp('200101/2', 'yyyyMM1/dd')`, the digit `1` in format string are ignored, and the output timestsamp is `2001-01-02 00:00:00`. Spaces and tabs in formats and tiemstamp string are also ignored automatically.
 - The precision of the output timestamp will be the same as the table in SELECT stmt, millisecond will be used if no table is specified. The output of `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')` will be truncated to millisecond precision. If a nano precision table is specified, no truncation will be applied. Like `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`.
--- a/docs/en/12-taos-sql/12-distinguished.md
+++ b/docs/en/12-taos-sql/12-distinguished.md
@ -38,11 +38,16 @@ Aggregation by time window is supported in TDengine. For example, in the case wh
 window_clause: {
    SESSION(ts_col, tol_val)
  | STATE_WINDOW(col)
-  | INTERVAL(interval [, offset]) [SLIDING sliding] [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})]
+  | INTERVAL(interval_val [, offset]) [SLIDING (sliding_value)] [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})]
  | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
 }
 ```

+Both interval_val and sliding_value are time durations which have 3 forms of representation.
+- INTERVAL(1s, 500a) SLIDING(1s), the unit char should be any one of a (millisecond), b (nanosecond), d (day), h (hour), m (minute), n (month), s (second), u (microsecond), w (week), y (year).
+- INTERVAL(1000, 500) SLIDING(1000), the unit will the same as the queried database, if there are more than one databases, higher precision will be used.
+- INTERVAL('1s', '500a') SLIDING('1s'), unit must be specified, no spaces allowed.
+
 The following restrictions apply:

 ### Other Rules
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@ -565,7 +565,7 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal)
 - `MONTH`, `MON`, `DAY`, `DY` 以及其他输出为数字的格式的大小写意义相同, 如 `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month`可以被替换为`MONTH` 或者`Month`.
 - 如果同一字段被指定了多次, 那么前面的指定将会被覆盖.  如 `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, 输出年份是`2022`.
 - 为避免转换时使用了非预期的时区，推荐在时间中携带时区信息，例如'2023-10-10 10:10:10+08'，如果未指定时区则默认时区为服务端或客户端指定的时区。
- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分.
+- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分. 暂不支持只指定年日而不指定月日的格式, 如'yyyy-mm-DDD', 支持'yyyy-mm-DD'.
 - 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12.
 - `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp('  23 年 - 1 月 - 01 日  ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换.
 - 输出时间戳的精度与查询表的精度相同, 若查询未指定表, 则输出精度为毫秒. 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')`的输出将会把微妙和纳秒进行截断. 如果指定一张纳秒表, 那么就不会发生截断, 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`.
--- a/docs/zh/12-taos-sql/12-distinguished.md
+++ b/docs/zh/12-taos-sql/12-distinguished.md
@ -44,7 +44,11 @@ window_clause: {
 }
 ```

-在上述语法中的具体限制如下
+其中，interval_val 和 sliding_val 都表示时间段, 语法上支持三种方式，举例说明如下:
+ - INTERVAL(1s, 500a) SLIDING(1s), 自带时间单位的形式，其中的时间单位是单字符表示, 分别为: a (毫秒), b (纳秒), d (天), h (小时), m (分钟), n (月), s (秒), u (微妙), w (周), y (年).
+ - INTERVAL(1000, 500) SLIDING(1000), 不带时间单位的形式，将使用查询库的时间精度作为默认时间单位，当存在多个库时默认采用精度更高的库.
+ - INTERVAL('1s', '500a') SLIDING('1s'), 自带时间单位的字符串形式，字符串内部不能有任何空格等其它字符.
+

 ### 窗口子句的规则

--- a/source/dnode/vnode/src/inc/vndCos.h
+++ b/source/dnode/vnode/src/inc/vndCos.h
@ -16,7 +16,7 @@
 #ifndef _TD_VND_COS_H_
 #define _TD_VND_COS_H_

-#include "vnd.h"
+#include "os.h"

 #ifdef __cplusplus
 extern "C" {
@ -24,6 +24,7 @@ extern "C" {

 #define S3_BLOCK_CACHE

+extern int8_t  tsS3StreamEnabled;
 extern int8_t  tsS3Enabled;
 extern int32_t tsS3BlockSize;
 extern int32_t tsS3BlockCacheSize;
@ -38,7 +39,8 @@ void    s3DeleteObjectsByPrefix(const char *prefix);
 void    s3DeleteObjects(const char *object_name[], int nobject);
 bool    s3Exists(const char *object_name);
 bool    s3Get(const char *object_name, const char *path);
-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock);
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
+int32_t s3GetObjectsByPrefix(const char *prefix, const char *path);
 void    s3EvictCache(const char *path, long object_size);
 long    s3Size(const char *object_name);

--- a/include/common/rsync.h
+++ b/include/common/rsync.h
@ -0,0 +1,24 @@
+//
+// Created by mingming wanng on 2023/11/2.
+//
+
+#ifndef TDENGINE_RSYNC_H
+#define TDENGINE_RSYNC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tarray.h"
+
+void stopRsync();
+void startRsync();
+int  uploadRsync(char* id, char* path);
+int  downloadRsync(char* id, char* path);
+int  deleteRsync(char* id);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_RSYNC_H
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@ -249,6 +249,7 @@ typedef struct SQueryTableDataCond {
  SColumnInfo* colList;
  int32_t*     pSlotList;  // the column output destation slot, and it may be null
  int32_t      type;       // data block load type:
+  bool         skipRollup;
  STimeWindow  twindows;
  int64_t      startVersion;
  int64_t      endVersion;
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@ -44,10 +44,6 @@ extern int32_t  tsNumOfSupportVnodes;
 extern int32_t tsMaxShellConns;
 extern int32_t tsShellActivityTimer;
 extern int32_t tsCompressMsgSize;
-extern int32_t tsCompressColData;
-extern int32_t tsMaxNumOfDistinctResults;
-extern int32_t tsCompatibleModel;
-extern bool    tsPrintAuth;
 extern int64_t tsTickPerMin[3];
 extern int64_t tsTickPerHour[3];
 extern int32_t tsCountAlwaysReturnValue;
@ -79,8 +75,12 @@ extern int32_t tsElectInterval;
 extern int32_t tsHeartbeatInterval;
 extern int32_t tsHeartbeatTimeout;

-// vnode
-extern int64_t tsVndCommitMaxIntervalMs;
+// snode
+extern int32_t tsRsyncPort;
+extern char tsCheckpointBackupDir[];
+
+// vnode checkpoint
+extern char tsSnodeAddress[];       //127.0.0.1:873

 // mnode
 extern int64_t tsMndSdbWriteDelta;
@ -105,8 +105,6 @@ extern bool     tsMonitorComp;

 // audit
 extern bool     tsEnableAudit;
-extern char     tsAuditFqdn[];
-extern uint16_t tsAuditPort;
 extern bool     tsEnableAuditCreateTable;

 // telem
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@ -49,6 +49,7 @@ typedef struct {
  uint64_t    checkpointId;
  bool        initTableReader;
  bool        initTqReader;
+  bool        skipRollup;
  int32_t     numOfVgroups;
  void*       sContext;  // SSnapContext*
  void*       pStateBackend;
@ -184,11 +185,7 @@ void qDestroyTask(qTaskInfo_t tinfo);

 void qProcessRspMsg(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);

-int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList /*,int32_t* resNum, SExplainExecInfo** pRes*/);
-
-int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len);
-
-int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len);
+int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList);

 void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order);
 void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery);
@ -217,7 +214,7 @@ int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRan
 int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow);
 int32_t qStreamRecoverFinish(qTaskInfo_t tinfo);
 int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo);
-bool    qStreamRecoverScanFinished(qTaskInfo_t tinfo);
+bool    qStreamScanhistoryFinished(qTaskInfo_t tinfo);
 int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo);
 void    resetTaskInfo(qTaskInfo_t tinfo);

--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@ -168,7 +168,6 @@ typedef struct {
  struct SStreamFileState *pFileState;
  int32_t                  number;
  SSHashObj               *parNameMap;
-  int64_t                  checkPointId;
  int32_t                  taskId;
  int64_t                  streamId;
  int64_t                  streamBackendRid;
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@ -388,6 +388,7 @@ typedef struct SLastRowScanPhysiNode {
  SNodeList*     pGroupTags;
  bool           groupSort;
  bool           ignoreNull;
+  SNodeList*     pTargets;
 } SLastRowScanPhysiNode;

 typedef SLastRowScanPhysiNode STableCountScanPhysiNode;
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@ -241,6 +241,24 @@ typedef struct {
  SEpSet  epset;
 } SDownstreamTaskEpset;

+typedef enum {
+  TASK_SCANHISTORY_CONT = 0x1,
+  TASK_SCANHISTORY_QUIT = 0x2,
+  TASK_SCANHISTORY_REXEC = 0x3,
+} EScanHistoryRet;
+
+typedef struct {
+  EScanHistoryRet ret;
+  int32_t         idleTime;
+} SScanhistoryDataInfo;
+
+typedef struct {
+  int32_t idleDuration;   // idle time before use time slice the continue execute scan-history
+  int32_t numOfTicks;
+  tmr_h   pTimer;
+  int32_t execCount;
+} SScanhistorySchedInfo;
+
 typedef struct {
  int64_t         stbUid;
  char            stbFullName[TSDB_TABLE_FNAME_LEN];
@ -354,7 +372,9 @@ typedef struct STaskExecStatisInfo {
  int64_t       init;
  int64_t       start;
  int64_t       step1Start;
+  double        step1El;
  int64_t       step2Start;
+  double        step2El;
  int32_t       updateCount;
  int64_t       latestUpdateTs;
  int32_t       processDataBlocks;
@ -378,6 +398,7 @@ typedef struct STaskOutputInfo {
  union {
    STaskDispatcherFixed   fixedDispatcher;
    STaskDispatcherShuffle shuffleDispatcher;
+
    STaskSinkTb    tbSink;
    STaskSinkSma   smaSink;
    STaskSinkFetch fetchSink;
@ -414,7 +435,10 @@ struct SStreamTask {
  SStreamState*       pState;         // state backend
  SArray*             pRspMsgList;
  SUpstreamInfo       upstreamInfo;
+
  // the followings attributes don't be serialized
+  SScanhistorySchedInfo schedHistoryInfo;
+
  int32_t             notReadyTasks;
  int32_t             numOfWaitingUpstream;
  int64_t             checkReqId;
@ -432,8 +456,10 @@ struct SStreamTask {
 typedef struct STaskStartInfo {
  int64_t   startTs;
  int64_t   readyTs;
-  int32_t   startAllTasksFlag;
+  int32_t   tasksWillRestart;
+  int32_t   taskStarting;          // restart flag, sentinel to guard the restart procedure.
  SHashObj* pReadyTaskSet;         // tasks that are all ready for running stream processing
+  SHashObj* pFailedTaskSet;        // tasks that are done the check downstream process, may be successful or failed
  int32_t   elapsedTime;
 } STaskStartInfo;

@ -732,8 +758,6 @@ void    initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen)

 // recover and fill history
 void    streamTaskCheckDownstream(SStreamTask* pTask);
-int32_t onNormalTaskReady(SStreamTask* pTask);
-int32_t onScanhistoryTaskReady(SStreamTask* pTask);

 int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage);
 int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
@ -755,7 +779,9 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs
 int32_t streamLaunchFillHistoryTask(SStreamTask* pTask);
 int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask);
 int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
+int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration);
 bool    streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
+
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);

 // common
@ -778,12 +804,11 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);
 // source level
 int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow);
 int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow);
-int32_t streamScanHistoryData(SStreamTask* pTask);
+SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
 int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask);

 // agg level
-int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq,
-                                          SRpcHandleInfo* pRpcInfo);
+int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, SRpcHandleInfo* pInfo);
 int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask);

 // stream task meta
@ -801,19 +826,22 @@ void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaReopen(SStreamMeta* pMeta);
 int32_t      streamMetaCommit(SStreamMeta* pMeta);
 int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
+int64_t      streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
 void         streamMetaStartHb(SStreamMeta* pMeta);
 void         streamMetaInitForSnode(SStreamMeta* pMeta);
 bool         streamMetaTaskInTimer(SStreamMeta* pMeta);
-int32_t      streamMetaUpdateTaskReadyInfo(SStreamTask* pTask);
+int32_t      streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool succ);
 void         streamMetaRLock(SStreamMeta* pMeta);
 void         streamMetaRUnLock(SStreamMeta* pMeta);
 void         streamMetaWLock(SStreamMeta* pMeta);
 void         streamMetaWUnLock(SStreamMeta* pMeta);
+void         streamMetaResetStartInfo(STaskStartInfo* pMeta);

 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
+int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask);
 int32_t streamAlignTransferState(SStreamTask* pTask);
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
--- a/include/os/osDef.h
+++ b/include/os/osDef.h
@ -237,6 +237,12 @@ void syslog(int unused, const char *format, ...);
 #define TD_DIRSEP "/"
 #endif

+#if defined(_WIN32)
+#define TD_DIRSEP_CHAR '\\'
+#else
+#define TD_DIRSEP_CHAR '/'
+#endif
+
 #define TD_LOCALE_LEN   64
 #define TD_CHARSET_LEN  64
 #define TD_TIMEZONE_LEN 96
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@ -754,6 +754,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_DUP_TIMESTAMP            TAOS_DEF_ERROR_CODE(0, 0x2805)
 #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806)
 #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
+#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808)

 //udf
 #define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)
--- a/include/util/tconfig.h
+++ b/include/util/tconfig.h
@ -57,10 +57,27 @@ typedef enum {
  CFG_SCOPE_BOTH
 } ECfgScopeType;

+typedef enum {
+  CFG_DYN_NONE = 0,
+  CFG_DYN_SERVER = 1,
+  CFG_DYN_CLIENT = 2,
+  CFG_DYN_BOTH = 3,
+#ifdef TD_ENTERPRISE
+  CFG_DYN_ENT_SERVER = CFG_DYN_SERVER,
+  CFG_DYN_ENT_CLIENT = CFG_DYN_CLIENT,
+  CFG_DYN_ENT_BOTH = CFG_DYN_BOTH,
+#else
+  CFG_DYN_ENT_SERVER = CFG_DYN_NONE,
+  CFG_DYN_ENT_CLIENT = CFG_DYN_NONE,
+  CFG_DYN_ENT_BOTH = CFG_DYN_NONE,
+#endif
+} ECfgDynType;
+
 typedef struct SConfigItem {
  ECfgSrcType  stype;
  ECfgDataType dtype;
  int8_t       scope;
+  int8_t       dynScope;
  char        *name;
  union {
    bool    bval;
@ -99,15 +116,20 @@ int32_t      cfgGetSize(SConfig *pCfg);
 SConfigItem *cfgGetItem(SConfig *pCfg, const char *name);
 int32_t      cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype);

-int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope);
-int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
-int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
-int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope);
-int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
-int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
-int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
-int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
-int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
+int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer);
+
+int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope);
+int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope,
+                    int8_t dynScope);
+int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope,
+                    int8_t dynScope);
+int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope,
+                    int8_t dynScope);
+int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
+int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
+int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
+int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);
+int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope);

 const char *cfgStypeStr(ECfgSrcType type);
 const char *cfgDtypeStr(ECfgDataType type);
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@ -1052,6 +1052,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
    }
  }

+  pRequest->body.execMode = pQuery->execMode;
  switch (pQuery->execMode) {
    case QUERY_EXEC_MODE_LOCAL:
      if (!pRequest->validateOnly) {
--- a/source/common/CMakeLists.txt
+++ b/source/common/CMakeLists.txt
@ -46,6 +46,75 @@ target_link_libraries(
    INTERFACE api
 )

+if(${BUILD_S3})
+
+    if(${BUILD_WITH_S3})
+        target_include_directories(
+                common
+
+                PUBLIC "$ENV{HOME}/.cos-local.2/include"
+        )
+
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+        set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
+        find_library(S3_LIBRARY s3)
+        find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
+        find_library(XML2_LIBRARY xml2)
+        find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
+        find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
+        target_link_libraries(
+                common
+
+                # s3
+                PUBLIC ${S3_LIBRARY}
+                PUBLIC ${CURL_LIBRARY}
+                PUBLIC ${SSL_LIBRARY}
+                PUBLIC ${CRYPTO_LIBRARY}
+                PUBLIC ${XML2_LIBRARY}
+        )
+
+        add_definitions(-DUSE_S3)
+    endif()
+
+    if(${BUILD_WITH_COS})
+
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+        find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
+        find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
+        find_library(MINIXML_LIBRARY mxml)
+        find_library(CURL_LIBRARY curl)
+        target_link_libraries(
+                common
+
+                # s3
+                PUBLIC cos_c_sdk_static
+                PUBLIC ${APR_UTIL_LIBRARY}
+                PUBLIC ${APR_LIBRARY}
+                PUBLIC ${MINIXML_LIBRARY}
+                PUBLIC ${CURL_LIBRARY}
+        )
+
+        # s3
+        FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
+        IF (APR_CONFIG_BIN)
+            EXECUTE_PROCESS(
+                    COMMAND ${APR_CONFIG_BIN} --includedir
+                    OUTPUT_VARIABLE APR_INCLUDE_DIR
+                    OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        ENDIF()
+        include_directories (${APR_INCLUDE_DIR})
+        target_include_directories(
+                common
+                PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
+                PUBLIC "$ENV{HOME}/.cos-local.1/include"
+        )
+
+        add_definitions(-DUSE_COS)
+    endif(${BUILD_WITH_COS})
+
+endif()
+
 if(${BUILD_TEST})
    ADD_SUBDIRECTORY(test)
 endif(${BUILD_TEST})
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/dnode/vnode/src/vnd/vnodeCos.c
@ -1,6 +1,6 @@
 #define ALLOW_FORBID_FUNC

-#include "vndCos.h"
+#include "cos.h"

 extern char   tsS3Endpoint[];
 extern char   tsS3AccessKeyId[];
@ -13,6 +13,7 @@ extern int8_t tsS3Https;
 #if defined(USE_S3)

 #include "libs3.h"
+#include "tarray.h"

 static int         verifyPeerG = 0;
 static const char *awsRegionG = NULL;
@ -24,7 +25,7 @@ static S3UriStyle uriStyleG = S3UriStylePath;
 static int        retriesG = 5;
 static int        timeoutMsG = 0;

-static int32_t s3Begin() {
+int32_t s3Begin() {
  S3Status    status;
  const char *hostname = tsS3Hostname;
  const char *env_hn = getenv("S3_HOSTNAME");
@ -34,7 +35,7 @@ static int32_t s3Begin() {
  }

  if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
-    vError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
+    uError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
    return -1;
  }

@ -43,10 +44,12 @@ static int32_t s3Begin() {
  return 0;
 }

-static void s3End() { S3_deinitialize(); }
-int32_t     s3Init() { return s3Begin(); }
+void s3End() { S3_deinitialize(); }

-void s3CleanUp() { s3End(); }
+int32_t s3Init() { return 0; /*s3Begin();*/ }
+
+void s3CleanUp() { /*s3End();*/
+}

 static int should_retry() {
  /*
@ -65,19 +68,32 @@ static int should_retry() {

 static void s3PrintError(const char *func, S3Status status, char error_details[]) {
  if (status < S3StatusErrorAccessDenied) {
-    vError("%s: %s", __func__, S3_get_status_name(status));
+    uError("%s: %s", __func__, S3_get_status_name(status));
  } else {
-    vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
+    uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
  }
 }

+typedef struct {
+  char      err_msg[512];
+  S3Status  status;
+  uint64_t  content_length;
+  TdFilePtr file;
+} TS3GetData;
+
 typedef struct {
  char     err_msg[128];
  S3Status status;
  uint64_t content_length;
  char    *buf;
+  int64_t  buf_pos;
 } TS3SizeCBD;

+static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *properties, void *callbackData) {
+  //  (void)callbackData;
+  return S3StatusOK;
+}
+
 static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
  //(void)callbackData;
  TS3SizeCBD *cbd = callbackData;
@ -97,24 +113,26 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
  int       len = 0;
  const int elen = sizeof(cbd->err_msg);
  if (error) {
-    if (error->message) {
+    if (error->message && elen - len > 0) {
      len += snprintf(&(cbd->err_msg[len]), elen - len, "  Message: %s\n", error->message);
    }
-    if (error->resource) {
+    if (error->resource && elen - len > 0) {
      len += snprintf(&(cbd->err_msg[len]), elen - len, "  Resource: %s\n", error->resource);
    }
-    if (error->furtherDetails) {
+    if (error->furtherDetails && elen - len > 0) {
      len += snprintf(&(cbd->err_msg[len]), elen - len, "  Further Details: %s\n", error->furtherDetails);
    }
-    if (error->extraDetailsCount) {
+    if (error->extraDetailsCount && elen - len > 0) {
      len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", "  Extra Details:\n");
      for (int i = 0; i < error->extraDetailsCount; i++) {
+        if (elen - len > 0) {
          len += snprintf(&(cbd->err_msg[len]), elen - len, "    %s: %s\n", error->extraDetails[i].name,
                          error->extraDetails[i].value);
        }
      }
    }
  }
+}

 typedef struct growbuffer {
  // The total number of bytes, and the start byte
@ -201,8 +219,9 @@ static void growbuffer_destroy(growbuffer *gb) {
 }

 typedef struct put_object_callback_data {
-  char     err_msg[128];
+  char     err_msg[512];
  S3Status status;
+  uint64_t content_length;
  // FILE       *infile;
  TdFilePtr   infileFD;
  growbuffer *gb;
@ -214,8 +233,9 @@ typedef struct put_object_callback_data {
 #define MULTIPART_CHUNK_SIZE (768 << 20)  // multipart is 768M

 typedef struct UploadManager {
-  char     err_msg[128];
+  char     err_msg[512];
  S3Status status;
+  uint64_t content_length;
  // used for initial multipart
  char *upload_id;

@ -229,8 +249,9 @@ typedef struct UploadManager {
 } UploadManager;

 typedef struct list_parts_callback_data {
-  char           err_msg[128];
+  char           err_msg[512];
  S3Status       status;
+  uint64_t       content_length;
  int            isTruncated;
  char           nextPartNumberMarker[24];
  char           initiatorId[256];
@ -246,7 +267,7 @@ typedef struct list_parts_callback_data {
 } list_parts_callback_data;

 typedef struct MultipartPartData {
-  char                     err_msg[128];
+  char                     err_msg[512];
  S3Status                 status;
  put_object_callback_data put_object_data;
  int                      seq;
@ -291,7 +312,7 @@ S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
 }

 S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) {
-  responsePropertiesCallback(properties, callbackData);
+  responsePropertiesCallbackNull(properties, callbackData);

  MultipartPartData *data = (MultipartPartData *)callbackData;
  int                seq = data->seq;
@ -390,7 +411,8 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
  S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
                                   0, awsRegionG};

-  S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback};
+  S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+&listPartsCallback};

  list_parts_callback_data data;

@ -445,13 +467,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
  data.noStatus = noStatus;

  if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
-    vError("ERROR: %s Failed to stat file %s: ", __func__, file);
+    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
    code = TAOS_SYSTEM_ERROR(errno);
    return code;
  }

  if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
-    vError("ERROR: %s Failed to open file %s: ", __func__, file);
+    uError("ERROR: %s Failed to open file %s: ", __func__, file);
    code = TAOS_SYSTEM_ERROR(errno);
    return code;
  }
@ -469,7 +491,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
                                   metaProperties,  useServerSideEncryption};

  if (contentLength <= MULTIPART_CHUNK_SIZE) {
-    S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+    S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                           &putObjectDataCallback};

    do {
@ -486,7 +508,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
      s3PrintError(__func__, data.status, data.err_msg);
      code = TAOS_SYSTEM_ERROR(EIO);
    } else if (data.contentLength) {
-      vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
+      uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
             (unsigned long long)data.contentLength);
      code = TAOS_SYSTEM_ERROR(EIO);
    }
@ -506,14 +528,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
    memset(&partData, 0, sizeof(MultipartPartData));
    int partContentLength = 0;

-    S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback},
+    S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                         &initial_multipart_callback};

    S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
                                           &putObjectDataCallback};

    S3MultipartCommitHandler commit_handler = {
-        {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+        {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};

    manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq);
    manager.next_etags_pos = 0;
@ -609,7 +631,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
 }

 typedef struct list_bucket_callback_data {
-  char     err_msg[128];
+  char     err_msg[512];
  S3Status status;
  int      isTruncated;
  char     nextMarker[1024];
@ -658,19 +680,19 @@ static void s3FreeObjectKey(void *pItem) {
  taosMemoryFree(key);
 }

-void s3DeleteObjectsByPrefix(const char *prefix) {
+static SArray *getListByPrefix(const char *prefix) {
  S3BucketContext     bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
                                       0, awsRegionG};
-  S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+  S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                           &listBucketCallback};

  const char               *marker = 0, *delimiter = 0;
  int                       maxkeys = 0, allDetails = 0;
  list_bucket_callback_data data;
-  data.objectArray = taosArrayInit(32, POINTER_BYTES);
+  data.objectArray = taosArrayInit(32, sizeof(void *));
  if (!data.objectArray) {
-    vError("%s: %s", __func__, "out of memoty");
-    return;
+    uError("%s: %s", __func__, "out of memoty");
+    return NULL;
  }
  if (marker) {
    snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
@ -693,18 +715,15 @@ void s3DeleteObjectsByPrefix(const char *prefix) {

  if (data.status == S3StatusOK) {
    if (data.keyCount > 0) {
-      // printListBucketHeader(allDetails);
-      s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray));
+      return data.objectArray;
    }
  } else {
    s3PrintError(__func__, data.status, data.err_msg);
  }
-
-  taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
+  return NULL;
 }

 void s3DeleteObjects(const char *object_name[], int nobject) {
-  int               status = 0;
  S3BucketContext   bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
                                     0, awsRegionG};
  S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
@ -721,17 +740,28 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
  }
 }

+void s3DeleteObjectsByPrefix(const char *prefix) {
+  SArray *objectArray = getListByPrefix(prefix);
+  if (objectArray == NULL) return;
+  s3DeleteObjects(TARRAY_DATA(objectArray), TARRAY_SIZE(objectArray));
+  taosArrayDestroyEx(objectArray, s3FreeObjectKey);
+}
+
 static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) {
  TS3SizeCBD *cbd = callbackData;
+  /*
  if (cbd->content_length != bufferSize) {
    cbd->status = S3StatusAbortedByCallback;
    return S3StatusAbortedByCallback;
  }
+  */
+  if (!cbd->buf) {
+    cbd->buf = taosMemoryCalloc(1, cbd->content_length);
+  }

-  char *buf = taosMemoryCalloc(1, bufferSize);
-  if (buf) {
-    memcpy(buf, buffer, bufferSize);
-    cbd->buf = buf;
+  if (cbd->buf) {
+    memcpy(cbd->buf + cbd->buf_pos, buffer, bufferSize);
+    cbd->buf_pos += bufferSize;
    cbd->status = S3StatusOK;
    return S3StatusOK;
  } else {
@ -740,7 +770,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *
  }
 }

-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) {
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
  int         status = 0;
  int64_t     ifModifiedSince = -1, ifNotModifiedSince = -1;
  const char *ifMatch = 0, *ifNotMatch = 0;
@ -753,12 +783,18 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,

  TS3SizeCBD cbd = {0};
  cbd.content_length = size;
+  cbd.buf_pos = 0;
  do {
    S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd);
  } while (S3_status_is_retryable(cbd.status) && should_retry());

  if (cbd.status != S3StatusOK) {
-    vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    return TAOS_SYSTEM_ERROR(EIO);
+  }
+
+  if (check && cbd.buf_pos != size) {
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
    return TAOS_SYSTEM_ERROR(EIO);
  }

@ -767,6 +803,66 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
  return 0;
 }

+static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) {
+  TS3GetData *cbd = (TS3GetData *)callbackData;
+  size_t      wrote = taosWriteFile(cbd->file, buffer, bufferSize);
+  return ((wrote < (size_t)bufferSize) ? S3StatusAbortedByCallback : S3StatusOK);
+}
+
+int32_t s3GetObjectToFile(const char *object_name, char *fileName) {
+  int64_t     ifModifiedSince = -1, ifNotModifiedSince = -1;
+  const char *ifMatch = 0, *ifNotMatch = 0;
+
+  S3BucketContext    bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                      0, awsRegionG};
+  S3GetConditions    getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
+  S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                         &getObjectCallback};
+
+  TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    uError("[s3] open file error, errno:%d, fileName:%s", errno, fileName);
+    return -1;
+  }
+  TS3GetData cbd = {0};
+  cbd.file = pFile;
+  do {
+    S3_get_object(&bucketContext, object_name, &getConditions, 0, 0, 0, 0, &getObjectHandler, &cbd);
+  } while (S3_status_is_retryable(cbd.status) && should_retry());
+
+  if (cbd.status != S3StatusOK) {
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    taosCloseFile(&pFile);
+    return TAOS_SYSTEM_ERROR(EIO);
+  }
+
+  taosCloseFile(&pFile);
+  return 0;
+}
+
+int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
+  SArray *objectArray = getListByPrefix(prefix);
+  if (objectArray == NULL) return -1;
+
+  for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
+    char       *object = taosArrayGetP(objectArray, i);
+    const char *tmp = strchr(object, '/');
+    tmp = (tmp == NULL) ? object : tmp + 1;
+    char fileName[PATH_MAX] = {0};
+    if (path[strlen(path) - 1] != TD_DIRSEP_CHAR) {
+      snprintf(fileName, PATH_MAX, "%s%s%s", path, TD_DIRSEP, tmp);
+    } else {
+      snprintf(fileName, PATH_MAX, "%s%s", path, tmp);
+    }
+    if (s3GetObjectToFile(object, fileName) != 0) {
+      taosArrayDestroyEx(objectArray, s3FreeObjectKey);
+      return -1;
+    }
+  }
+  taosArrayDestroyEx(objectArray, s3FreeObjectKey);
+  return 0;
+}
+
 long s3Size(const char *object_name) {
  long size = 0;
  int  status = 0;
@ -782,7 +878,7 @@ long s3Size(const char *object_name) {
  } while (S3_status_is_retryable(cbd.status) && should_retry());

  if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-    vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
  }

  size = cbd.content_length;
@ -1045,7 +1141,8 @@ bool s3Get(const char *object_name, const char *path) {
  return ret;
 }

-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, uint8_t **ppBlock) {
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
+  (void)check;
  int32_t                code = 0;
  cos_pool_t            *p = NULL;
  int                    is_cname = 0;
@ -1237,8 +1334,11 @@ void    s3DeleteObjectsByPrefix(const char *prefix) {}
 void    s3DeleteObjects(const char *object_name[], int nobject) {}
 bool    s3Exists(const char *object_name) { return false; }
 bool    s3Get(const char *object_name, const char *path) { return false; }
-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { return 0; }
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
+  return 0;
+}
 void    s3EvictCache(const char *path, long object_size) {}
 long    s3Size(const char *object_name) { return 0; }
+int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { return 0; }

 #endif
--- a/source/common/src/rsync.c
+++ b/source/common/src/rsync.c
@ -0,0 +1,235 @@
+//
+// Created by mingming wanng on 2023/11/2.
+//
+#include "rsync.h"
+#include <stdlib.h>
+#include "tglobal.h"
+
+#define ERRNO_ERR_FORMAT  "errno:%d,msg:%s"
+#define ERRNO_ERR_DATA    errno,strerror(errno)
+
+// deleteRsync function produce empty directories, traverse base directory to remove them
+static void removeEmptyDir(){
+  TdDirPtr pDir = taosOpenDir(tsCheckpointBackupDir);
+  if (pDir == NULL) return;
+
+  TdDirEntryPtr de = NULL;
+  while ((de = taosReadDir(pDir)) != NULL) {
+    if (!taosDirEntryIsDir(de)) {
+      continue;
+    }
+
+    if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
+
+    char filename[PATH_MAX] = {0};
+    snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de));
+
+    TdDirPtr pDirTmp = taosOpenDir(filename);
+    TdDirEntryPtr deTmp = NULL;
+    bool empty = true;
+    while ((deTmp = taosReadDir(pDirTmp)) != NULL){
+      if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue;
+      empty = false;
+    }
+    if(empty) taosRemoveDir(filename);
+    taosCloseDir(&pDirTmp);
+  }
+
+  taosCloseDir(&pDir);
+}
+
+#ifdef WINDOWS
+// C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/
+static void changeDirFromWindowsToLinux(char* from, char* to){
+  to[0] = '/';
+  to[1] = from[0];
+  for(int i = 2; i < strlen(from); i++) {
+    if (from[i] == '\\') {
+      to[i] = '/';
+    } else {
+      to[i] = from[i];
+    }
+  }
+}
+#endif
+
+static int generateConfigFile(char* confDir){
+  TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    uError("[rsync] open conf file error, dir:%s,"ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
+    return -1;
+  }
+
+#ifdef WINDOWS
+  char path[PATH_MAX] = {0};
+  changeDirFromWindowsToLinux(tsCheckpointBackupDir, path);
+#endif
+
+  char confContent[PATH_MAX*4] = {0};
+  snprintf(confContent, PATH_MAX*4,
+#ifndef WINDOWS
+           "uid = root\n"
+           "gid = root\n"
+#endif
+           "use chroot = false\n"
+           "max connections = 200\n"
+           "timeout = 100\n"
+           "lock file = %srsync.lock\n"
+           "log file = %srsync.log\n"
+           "ignore errors = true\n"
+           "read only = false\n"
+           "list = false\n"
+           "[checkpoint]\n"
+           "path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir,
+#ifdef WINDOWS
+           path
+#else
+           tsCheckpointBackupDir
+#endif
+           );
+  uDebug("[rsync] conf:%s", confContent);
+  if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){
+    uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
+    taosCloseFile(&pFile);
+    return -1;
+  }
+
+  taosCloseFile(&pFile);
+  return 0;
+}
+
+static int execCommand(char* command){
+  int try = 3;
+  int32_t code = 0;
+  while(try-- > 0) {
+    code = system(command);
+    if (code == 0) {
+      break;
+    }
+    taosMsleep(10);
+  }
+  return code;
+}
+
+void stopRsync(){
+  int code =
+#ifdef WINDOWS
+  system("taskkill /f /im rsync.exe");
+#else
+  system("pkill rsync");
+#endif
+  if(code != 0){
+    uError("[rsync] stop rsync server failed,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
+    return;
+  }
+  uDebug("[rsync] stop rsync server successful");
+}
+
+void startRsync(){
+  if(taosMulMkDir(tsCheckpointBackupDir) != 0){
+    uError("[rsync] build checkpoint backup dir failed, dir:%s,"ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA);
+    return;
+  }
+  removeEmptyDir();
+
+  char confDir[PATH_MAX] = {0};
+  snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir);
+
+  int code = generateConfigFile(confDir);
+  if(code != 0){
+    return;
+  }
+
+  char cmd[PATH_MAX] = {0};
+  snprintf(cmd, PATH_MAX, "rsync --daemon --port=%d --config=%s", tsRsyncPort, confDir);
+  // start rsync service to backup checkpoint
+  code = system(cmd);
+  if(code != 0){
+    uError("[rsync] start server failed, code:%d,"ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return;
+  }
+  uDebug("[rsync] start server successful");
+}
+
+int uploadRsync(char* id, char* path){
+#ifdef WINDOWS
+  char pathTransform[PATH_MAX] = {0};
+  changeDirFromWindowsToLinux(path, pathTransform);
+#endif
+  char command[PATH_MAX] = {0};
+#ifdef WINDOWS
+  if(pathTransform[strlen(pathTransform) - 1] != '/'){
+#else
+  if(path[strlen(path) - 1] != '/'){
+#endif
+    snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/",
+#ifdef WINDOWS
+             pathTransform
+#else
+             path
+#endif
+             , tsSnodeAddress, id);
+  }else{
+    snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/",
+#ifdef WINDOWS
+             pathTransform
+#else
+             path
+#endif
+             , tsSnodeAddress, id);
+  }
+
+  int code = execCommand(command);
+  if(code != 0){
+    uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  uDebug("[rsync] upload data:%s successful", id);
+  return 0;
+}
+
+int downloadRsync(char* id, char* path){
+#ifdef WINDOWS
+  char pathTransform[PATH_MAX] = {0};
+  changeDirFromWindowsToLinux(path, pathTransform);
+#endif
+  char command[PATH_MAX] = {0};
+  snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
+           tsSnodeAddress, id,
+#ifdef WINDOWS
+           pathTransform
+#else
+           path
+#endif
+           );
+
+  int code = execCommand(command);
+  if(code != 0){
+    uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  uDebug("[rsync] down data:%s successful", id);
+  return 0;
+}
+
+int deleteRsync(char* id){
+  char* tmp = "./tmp_empty/";
+  int code = taosMkDir(tmp);
+  if(code != 0){
+    uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  char command[PATH_MAX] = {0};
+  snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/",
+           tmp, tsSnodeAddress, id);
+
+  code = execCommand(command);
+  taosRemoveDir(tmp);
+  if(code != 0){
+    uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  uDebug("[rsync] delete data:%s successful", id);
+
+  return 0;
+}
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
--- a/source/common/src/ttime.c
+++ b/source/common/src/ttime.c
@ -1580,6 +1580,7 @@ static bool needMoreDigits(SArray* formats, int32_t curIdx) {
 /// @retval 0 for success
 /// @retval -1 for format and s mismatch error
 /// @retval -2 if datetime err, like 2023-13-32 25:61:69
+/// @retval -3 if not supported
 static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t precision, const char** sErrPos,
                       int32_t* fErrIdx) {
  int32_t size = taosArrayGetSize(formats);
@ -1589,6 +1590,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
  int32_t hour = 0, min = 0, sec = 0, us = 0, ms = 0, ns = 0;
  int32_t tzSign = 1, tz = tsTimezone;
  int32_t err = 0;
+  bool    withYD = false, withMD = false;

  for (int32_t i = 0; i < size && *s != '\0'; ++i) {
    while (isspace(*s) && *s != '\0') {
@ -1782,6 +1784,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
        } else {
          s = newPos;
        }
+        withYD = true;
      } break;
      case TSFKW_DD: {
        const char* newPos = tsFormatStr2Int32(&md, s, 2, needMoreDigits(formats, i));
@ -1790,6 +1793,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
        } else {
          s = newPos;
        }
+        withMD = true;
      } break;
      case TSFKW_D: {
        const char* newPos = tsFormatStr2Int32(&wd, s, 1, needMoreDigits(formats, i));
@ -1843,6 +1847,10 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
      return err;
    }
  }
+  if (!withMD) {
+    // yyyy-mm-DDD, currently, the c api can't convert to correct timestamp, return not supported
+    if (withYD) return -3;
+  }
  struct STm tm = {0};
  tm.tm.tm_year = year - 1900;
  tm.tm.tm_mon = mon;
@ -1892,8 +1900,13 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int
    TSFormatNode* fNode = (taosArrayGet(*formats, fErrIdx));
    snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos,
             fErrIdx < taosArrayGetSize(*formats) ? ((TSFormatNode*)taosArrayGet(*formats, fErrIdx))->key->name : "");
+    code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR;
  } else if (code == -2) {
    snprintf(errMsg, errMsgLen, "timestamp format error: %s -> %s", tsStr, format);
+    code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR;
+  } else if (code == -3) {
+    snprintf(errMsg, errMsgLen, "timestamp format not supported");
+    code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED;
  }
  return code;
 }
--- a/source/common/test/commonTests.cpp
+++ b/source/common/test/commonTests.cpp
@ -489,6 +489,7 @@ TEST(timeTest, char2ts) {
  ASSERT_EQ(ts, 0);
  ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a a a 1/1+0"));
  ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a a a a a a a a a a a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a     "));
+  ASSERT_EQ(-3, TEST_char2ts("yyyy-mm-DDD", &ts, TSDB_TIME_PRECISION_MILLI, "1970-01-001"));
 }

 #pragma GCC diagnostic pop
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@ -144,7 +144,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
  char path[TSDB_FILENAME_LEN] = {0};
  bool atExit = true;

-  if (vnodeIsLeader(pVnode->pImpl)) {
+  if (pVnode->pImpl && vnodeIsLeader(pVnode->pImpl)) {
    vnodeProposeCommitOnNeed(pVnode->pImpl, atExit);
  }

@ -153,6 +153,10 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
  taosThreadRwlockUnlock(&pMgmt->lock);
  vmReleaseVnode(pMgmt, pVnode);

+  if (pVnode->failed) {
+    ASSERT(pVnode->pImpl == NULL);
+    goto _closed;
+  }
  dInfo("vgId:%d, pre close", pVnode->vgId);
  vnodePreClose(pVnode->pImpl);

@ -202,6 +206,8 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)

  vnodeClose(pVnode->pImpl);
  pVnode->pImpl = NULL;
+
+_closed:
  dInfo("vgId:%d, vnode is closed", pVnode->vgId);

  if (commitAndRemoveWal) {
@ -386,7 +392,6 @@ static void *vmCloseVnodeInThread(void *param) {

  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
    SVnodeObj *pVnode = pThread->ppVnodes[v];
-    if (pVnode->failed) continue;

    char stepDesc[TSDB_STEP_DESC_LEN] = {0};
    snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to close, %d of %d have been closed", pVnode->vgId,
--- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt
+++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt
@ -3,6 +3,17 @@ add_library(dnode STATIC ${IMPLEMENT_SRC})
 target_link_libraries(
    dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode
 )
+
+IF (TD_STORAGE)
+
+  IF(${BUILD_WITH_S3})
+    add_definitions(-DUSE_S3)
+  ELSEIF(${BUILD_WITH_COS})
+    add_definitions(-DUSE_COS)
+  ENDIF()
+
+ENDIF ()
+
 target_include_directories(
    dnode
    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
--- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
@ -146,6 +146,13 @@ static bool dmCheckDataDirVersion() {
  return true;
 }

+#if defined(USE_S3)
+
+extern int32_t s3Begin();
+extern void    s3End();
+
+#endif
+
 int32_t dmInit() {
  dInfo("start to init dnode env");
  if (dmDiskInit() != 0) return -1;
@ -156,6 +163,9 @@ int32_t dmInit() {
  if (dmInitMonitor() != 0) return -1;
  if (dmInitAudit() != 0) return -1;
  if (dmInitDnode(dmInstance()) != 0) return -1;
+#if defined(USE_S3)
+  if (s3Begin() != 0) return -1;
+#endif

  dInfo("dnode env is initialized");
  return 0;
@ -181,6 +191,9 @@ void dmCleanup() {
  udfStopUdfd();
  taosStopCacheRefreshWorker();
  dmDiskClose();
+#if defined(USE_S3)
+  s3End();
+#endif
  dInfo("dnode env is cleaned up");

  taosCleanupCfg();
@ -394,7 +407,4 @@ void dmReportStartup(const char *pName, const char *pDesc) {
  dDebug("step:%s, %s", pStartup->name, pStartup->desc);
 }

-int64_t dmGetClusterId() {
-  return globalDnode.data.clusterId;
-}
-
+int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@ -79,7 +79,7 @@ static void    mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
 static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextDnode(SMnode *pMnode, void *pIter);

-static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t opLen, int32_t *pOutValue);
+static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue);

 #ifdef _GRANT
 int32_t mndUpdClusterInfo(SRpcMsg *pReq);
@ -1182,15 +1182,72 @@ _OVER:
  return code;
 }

+static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) {
+  terrno = 0;
+  char *p = pMCfgReq->config;
+  while (*p) {
+    if (*p == ' ') {
+      break;
+    }
+    p++;
+  }
+
+  size_t optLen = p - pMCfgReq->config;
+  strncpy(pDCfgReq->config, pMCfgReq->config, optLen);
+  pDCfgReq->config[optLen] = 0;
+
+  if (' ' == pMCfgReq->config[optLen]) {
+    // 'key value'
+    if (strlen(pMCfgReq->value) != 0) goto _err;
+    strcpy(pDCfgReq->value, p + 1);
+  } else {
+    // 'key' 'value'
+    if (strlen(pMCfgReq->value) == 0) goto _err;
+    strcpy(pDCfgReq->value, pMCfgReq->value);
+  }
+
+  return 0;
+
+_err:
+  mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config);
+  terrno = TSDB_CODE_INVALID_CFG;
+  return -1;
+}
+
+static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) {
+  int32_t code = -1;
+  SSdb   *pSdb = pMnode->pSdb;
+  void   *pIter = NULL;
+  while (1) {
+    SDnodeObj *pDnode = NULL;
+    pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
+    if (pIter == NULL) break;
+
+    if (pDnode->id == dnodeId || dnodeId == -1 || dnodeId == 0) {
+      SEpSet  epSet = mndGetDnodeEpset(pDnode);
+      int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq);
+      void   *pBuf = rpcMallocCont(bufLen);
+
+      if (pBuf != NULL) {
+        tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq);
+        mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value);
+        SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen};
+        tmsgSendReq(&epSet, &rpcMsg);
+        code = 0;
+      }
+    }
+
+    sdbRelease(pSdb, pDnode);
+  }
+
+  if (code == -1) {
+    terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+  return code;
+}
+
 static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
  SMnode       *pMnode = pReq->info.node;
-  const char *options[] = {
-      "debugFlag",   "dDebugFlag",   "vDebugFlag",   "mDebugFlag",   "wDebugFlag",    "sDebugFlag",   "tsdbDebugFlag",
-      "tqDebugFlag", "fsDebugFlag",  "udfDebugFlag", "smaDebugFlag", "idxDebugFlag",  "tdbDebugFlag", "tmrDebugFlag",
-      "uDebugFlag",  "smaDebugFlag", "rpcDebugFlag", "qDebugFlag",   "metaDebugFlag", "stDebugFlag",
-  };
-  int32_t optionSize = tListLen(options);
-
  SMCfgDnodeReq cfgReq = {0};
  if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
    terrno = TSDB_CODE_INVALID_MSG;
@ -1206,139 +1263,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
  SDCfgDnodeReq dcfgReq = {0};
  if (strcasecmp(cfgReq.config, "resetlog") == 0) {
    strcpy(dcfgReq.config, "resetlog");
-  } else if (strncasecmp(cfgReq.config, "monitor", 7) == 0) {
-    if (' ' != cfgReq.config[7] && 0 != cfgReq.config[7]) {
-      mError("dnode:%d, failed to config monitor since invalid conf:%s", cfgReq.dnodeId, cfgReq.config);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    const char *value = cfgReq.value;
-    int32_t     flag = atoi(value);
-    if (flag <= 0) {
-      flag = atoi(cfgReq.config + 8);
-    }
-    if (flag < 0 || flag > 2) {
-      mError("dnode:%d, failed to config monitor since value:%d", cfgReq.dnodeId, flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "monitor");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
-    int32_t optLen = strlen("s3blocksize");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag > 1024 * 1024) {
-      mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId,
-             flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "s3blocksize");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, "s3blockcachesize", 16) == 0) {
-    int32_t optLen = strlen("s3blockcachesize");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 4 || flag > 1024 * 1024) {
-      mError("dnode:%d, failed to config s3BlockCacheSize since value:%d. Valid range: [4, 1024 * 1024]",
-             cfgReq.dnodeId, flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "s3blockcachesize");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, "s3pagecachesize", 16) == 0) {
-    int32_t optLen = strlen("s3pagecachesize");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 4 || flag > 1024 * 1024 * 1024) {
-      mError("dnode:%d, failed to config s3PageCacheSize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId,
-             flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "s3pagecachesize");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, "s3uploaddelaysec", 16) == 0) {
-    int32_t optLen = strlen("s3uploaddelaysec");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 600 || flag > 60 * 60 * 24 * 30) {
-      mError("dnode:%d, failed to config s3UploadDelaySec since value:%d. Valid range: [600, 60 * 60 * 24 * 30]",
-             cfgReq.dnodeId, flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "s3uploaddelaysec");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) {
-    int32_t optLen = strlen("ttlpushinterval");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 0 || flag > 100000) {
-      mError("dnode:%d, failed to config ttlPushInterval since value:%d. Valid range: [0, 100000]", cfgReq.dnodeId,
-             flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "ttlpushinterval");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, "ttlbatchdropnum", 15) == 0) {
-    int32_t optLen = strlen("ttlbatchdropnum");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 0) {
-      mError("dnode:%d, failed to config ttlBatchDropNum since value:%d. Valid range: [0, %d]", cfgReq.dnodeId, flag,
-             INT32_MAX);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "ttlbatchdropnum");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, "asynclog", 8) == 0) {
-    int32_t optLen = strlen("asynclog");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 0 || flag > 1) {
-      mError("dnode:%d, failed to config asynclog since value:%d. Valid range: [0, 1]", cfgReq.dnodeId, flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
-
-    strcpy(dcfgReq.config, "asynclog");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
 #ifdef TD_ENTERPRISE
  } else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) {
    int32_t optLen = strlen("supportvnodes");
@ -1348,9 +1272,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {

    if (flag < 0 || flag > 4096) {
      mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag);
-      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
+      terrno = TSDB_CODE_OUT_OF_RANGE;
+      goto _err_out;
    }
    if (flag == 0) {
      flag = tsNumOfCores * 2;
@ -1365,8 +1288,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
    if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) {
      mError("dnode:%d, failed to config activeCode since invalid conf:%s", cfgReq.dnodeId, cfgReq.config);
      terrno = TSDB_CODE_INVALID_CFG;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
+      goto _err_out;
    }
    int32_t vlen = strlen(cfgReq.value);
    if (vlen > 0 && ((opt == DND_ACTIVE_CODE && vlen != (TSDB_ACTIVE_KEY_LEN - 1)) ||
@ -1374,9 +1296,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
                      (vlen > (TSDB_CONN_ACTIVE_KEY_LEN - 1) || vlen < (TSDB_ACTIVE_KEY_LEN - 1))))) {
      mError("dnode:%d, failed to config activeCode since invalid vlen:%d. conf:%s, val:%s", cfgReq.dnodeId, vlen,
             cfgReq.config, cfgReq.value);
-      terrno = TSDB_CODE_INVALID_OPTION;
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
+      terrno = TSDB_CODE_INVALID_CFG;
+      goto _err_out;
    }

    strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode");
@ -1384,88 +1305,53 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {

    if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) {
      mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr());
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
+      terrno = TSDB_CODE_INVALID_CFG;
+      goto _err_out;
    }
    tFreeSMCfgDnodeReq(&cfgReq);
    return 0;
+  } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
+    int32_t optLen = strlen("s3blocksize");
+    int32_t flag = -1;
+    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
+    if (code < 0) return code;
+
+    if (flag > 1024 * 1024 || (flag > -1 && flag < 4) || flag < -1) {
+      mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [4, 1024 * 1024]",
+             cfgReq.dnodeId, flag);
+      terrno = TSDB_CODE_INVALID_CFG;
+      tFreeSMCfgDnodeReq(&cfgReq);
+      return -1;
+    }
+
+    strcpy(dcfgReq.config, "s3blocksize");
+    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
 #endif
  } else {
-    bool findOpt = false;
-    for (int32_t d = 0; d < optionSize; ++d) {
-      const char *optName = options[d];
-      int32_t     optLen = strlen(optName);
-      if (strncasecmp(cfgReq.config, optName, optLen) != 0) continue;
-
-      if (' ' != cfgReq.config[optLen] && 0 != cfgReq.config[optLen]) {
-        mError("dnode:%d, failed to config since invalid conf:%s", cfgReq.dnodeId, cfgReq.config);
+    mndMCfg2DCfg(&cfgReq, &dcfgReq);
+    if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) {
+      mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId);
      terrno = TSDB_CODE_INVALID_CFG;
-        tFreeSMCfgDnodeReq(&cfgReq);
-        return -1;
+      goto _err_out;
    }

-      const char *value = cfgReq.value;
-      int32_t     flag = atoi(value);
-      if (flag <= 0) {
-        flag = atoi(cfgReq.config + optLen + 1);
-      }
-      if (flag < 0 || flag > 255) {
-        mError("dnode:%d, failed to config %s since value:%d", cfgReq.dnodeId, optName, flag);
-        terrno = TSDB_CODE_INVALID_CFG;
-        tFreeSMCfgDnodeReq(&cfgReq);
-        return -1;
-      }
-
-      tstrncpy(dcfgReq.config, optName, optLen + 1);
-      snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-      findOpt = true;
-    }
-
-    if (!findOpt) {
-      terrno = TSDB_CODE_INVALID_CFG;
-      mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr());
-      tFreeSMCfgDnodeReq(&cfgReq);
-      return -1;
-    }
+    if (cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true) != 0) goto _err_out;
  }

+  {  // audit
    char obj[50] = {0};
    sprintf(obj, "%d", cfgReq.dnodeId);

-  auditRecord(pReq, pMnode->clusterId, "alterDnode", "", obj, cfgReq.sql, cfgReq.sqlLen);
+    auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen);
+  }

  tFreeSMCfgDnodeReq(&cfgReq);

-  int32_t code = -1;
-  SSdb   *pSdb = pMnode->pSdb;
-  void   *pIter = NULL;
-  while (1) {
-    SDnodeObj *pDnode = NULL;
-    pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
-    if (pIter == NULL) break;
+  return mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq);

-    if (pDnode->id == cfgReq.dnodeId || cfgReq.dnodeId == -1 || cfgReq.dnodeId == 0) {
-      SEpSet  epSet = mndGetDnodeEpset(pDnode);
-      int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq);
-      void   *pBuf = rpcMallocCont(bufLen);
-
-      if (pBuf != NULL) {
-        tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq);
-        mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle,
-              dcfgReq.config, dcfgReq.value);
-        SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen};
-        tmsgSendReq(&epSet, &rpcMsg);
-        code = 0;
-      }
-    }
-
-    sdbRelease(pSdb, pDnode);
-  }
-
-  if (code == -1) {
-    terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
-  }
-  return code;
+_err_out:
+  tFreeSMCfgDnodeReq(&cfgReq);
+  return -1;
 }

 static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) {
@ -1606,16 +1492,16 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) {
 }

 // get int32_t value from 'SMCfgDnodeReq'
-static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t opLen, int32_t *pOutValue) {
+static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) {
  terrno = 0;
-  if (' ' != pMCfgReq->config[opLen] && 0 != pMCfgReq->config[opLen]) {
+  if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) {
    goto _err;
  }

-  if (' ' == pMCfgReq->config[opLen]) {
+  if (' ' == pMCfgReq->config[optLen]) {
    // 'key value'
    if (strlen(pMCfgReq->value) != 0) goto _err;
-    *pOutValue = atoi(pMCfgReq->config + opLen + 1);
+    *pOutValue = atoi(pMCfgReq->config + optLen + 1);
  } else {
    // 'key' 'value'
    if (strlen(pMCfgReq->value) == 0) goto _err;
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@ -862,8 +862,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {

  // TODO
  if (createStreamReq.sql != NULL) {
-    auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, 
-                createStreamReq.sql, strlen(createStreamReq.sql));
+    auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, createStreamReq.sql,
+                strlen(createStreamReq.sql));
  }
 _OVER:
  if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@ -877,6 +877,21 @@ _OVER:
  return code;
 }

+int64_t mndStreamGenChkpId(SMnode *pMnode) {
+  SStreamObj *pStream = NULL;
+  void *      pIter = NULL;
+  SSdb *      pSdb = pMnode->pSdb;
+  int64_t     maxChkpId = 0;
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
+    if (pIter == NULL) break;
+
+    maxChkpId = TMAX(maxChkpId, pStream->checkpointId);
+    sdbRelease(pSdb, pStream);
+  }
+  return maxChkpId + 1;
+}
+
 static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
  SMnode *pMnode = pReq->info.node;
  SSdb *  pSdb = pMnode->pSdb;
@ -885,7 +900,7 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
  }

  SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
-  pMsg->checkpointId = taosGetTimestampMs();
+  pMsg->checkpointId = mndStreamGenChkpId(pMnode);

  int32_t size = sizeof(SMStreamDoCheckpointMsg);
  SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@ -1070,7 +1085,8 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream

        STransAction action = {0};
        SEpSet       epset = mndGetVgroupEpset(pMnode, pVgObj);
-        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
+        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset,
+                        TSDB_CODE_SYN_PROPOSE_NOT_READY);
        mndReleaseVgroup(pMnode, pVgObj);

        if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@ -1160,8 +1176,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
    bool    allReady = true;
    SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
    if (!allReady) {
-      mWarn("not all vnodes are ready, ignore the checkpoint")
-      taosArrayDestroy(pNodeSnapshot);
+      mWarn("not all vnodes are ready, ignore the checkpoint") taosArrayDestroy(pNodeSnapshot);
      return 0;
    }

@ -2361,7 +2376,6 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {

  SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
  if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) {
-
    // kill current active checkpoint transaction, since the transaction is vnode wide.
    doKillActiveCheckpointTrans(pMnode);
    code = mndProcessVgroupChange(pMnode, &changeInfo);
@ -2457,7 +2471,6 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
            break;
          }
        }
-
      }
    }
  }
@ -2634,7 +2647,6 @@ static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) {
  for (int32_t j = 0; j < numOfNodes; ++j) {
    SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j);
    if (pNodeEntry->nodeId == pTaskEntry->nodeId) {
-
      mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId,
            pTaskEntry->stage, stage, pTaskEntry->id.taskId);

--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@ -13,6 +13,7 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "rsync.h"
 #include "executor.h"
 #include "sndInt.h"
 #include "tstream.h"
@ -122,6 +123,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
    goto FAIL;
  }

+  stopRsync();
+  startRsync();
+
  // todo fix it: send msg to mnode to rollback to an existed checkpoint
  streamMetaInitForSnode(pSnode->pMeta);
  return pSnode;
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@ -8,7 +8,6 @@ set(
  "src/vnd/vnodeCommit.c"
  "src/vnd/vnodeQuery.c"
  "src/vnd/vnodeModule.c"
-  "src/vnd/vnodeCos.c"
  "src/vnd/vnodeSvr.c"
  "src/vnd/vnodeSync.c"
  "src/vnd/vnodeSnapshot.c"
@ -161,75 +160,6 @@ target_link_libraries(
    PUBLIC index
 )

-if(${BUILD_S3})
-
-if(${BUILD_WITH_S3})
-  target_include_directories(
-    vnode
-
-    PUBLIC "$ENV{HOME}/.cos-local.2/include"
-    )
-
-  set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-  set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
-  find_library(S3_LIBRARY s3)
-  find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
-  find_library(XML2_LIBRARY xml2)
-  find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
-  find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
-  target_link_libraries(
-    vnode
-
-    # s3
-    PUBLIC ${S3_LIBRARY}
-    PUBLIC ${CURL_LIBRARY}
-    PUBLIC ${SSL_LIBRARY}
-    PUBLIC ${CRYPTO_LIBRARY}
-    PUBLIC ${XML2_LIBRARY}
-  )
-
-  add_definitions(-DUSE_S3)
-endif()
-
-if(${BUILD_WITH_COS})
-
-set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
-find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
-find_library(MINIXML_LIBRARY mxml)
-find_library(CURL_LIBRARY curl)
-target_link_libraries(
-    vnode
-
-    # s3
-    PUBLIC cos_c_sdk_static
-    PUBLIC ${APR_UTIL_LIBRARY}
-    PUBLIC ${APR_LIBRARY}
-    PUBLIC ${MINIXML_LIBRARY}
-    PUBLIC ${CURL_LIBRARY}
-)
-
-# s3
-FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
-IF (APR_CONFIG_BIN)
-  EXECUTE_PROCESS(
-    COMMAND ${APR_CONFIG_BIN} --includedir
-    OUTPUT_VARIABLE APR_INCLUDE_DIR
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-ENDIF()
-include_directories (${APR_INCLUDE_DIR})
-target_include_directories(
-  vnode
-  PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
-  PUBLIC "$ENV{HOME}/.cos-local.1/include"
-  )
-
-  add_definitions(-DUSE_COS)
-endif(${BUILD_WITH_COS})
-
-endif()
-
 IF (TD_GRANT)
  TARGET_LINK_LIBRARIES(vnode PUBLIC grant)
 ENDIF ()
--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@ -108,6 +108,7 @@ struct SRSmaStat {
  int64_t          refId;        // shared by fetch tasks
  volatile int64_t nBufItems;    // number of items in queue buffer
  SRWLatch         lock;         // r/w lock for rsma fs(e.g. qtaskinfo)
+  volatile int32_t execStat;     // 0 succeed, other failed
  volatile int32_t nFetchAll;    // active number of fetch all
  volatile int8_t  triggerStat;  // shared by fetch tasks
  volatile int8_t  commitStat;   // 0 not in committing, 1 in committing
@ -115,6 +116,7 @@ struct SRSmaStat {
  SRSmaFS          fs;           // for recovery/snapshot r/w
  SHashObj        *infoHash;     // key: suid, value: SRSmaInfo
  tsem_t           notEmpty;     // has items in queue buffer
+  SArray          *blocks;       // SArray<SSDataBlock>
 };

 struct SSmaStat {
@ -136,13 +138,18 @@ struct SSmaStat {
 #define RSMA_FS_LOCK(r)      (&(r)->lock)

 struct SRSmaInfoItem {
-  int8_t   level : 4;
-  int8_t   fetchLevel : 4;
+  int8_t  level;
+  int8_t  fetchLevel;
  int8_t  triggerStat;
-  uint16_t nScanned;
-  int32_t  maxDelay;  // ms
+  int32_t nScanned;
+  int32_t streamFlushed : 1;
+  int32_t maxDelay : 31;  // ms
+  int64_t submitReqVer;
+  int64_t fetchResultVer;
  tmr_h   tmrId;
  void   *pStreamState;
+  void   *pStreamTask;  // SStreamTask
+  SArray *pResList;
 };

 struct SRSmaInfo {
@ -160,11 +167,9 @@ struct SRSmaInfo {
  STaosQall    *qall;                         // buffer qall of SubmitReq
 };

-#define RSMA_INFO_HEAD_LEN     offsetof(SRSmaInfo, items)
 #define RSMA_INFO_IS_DEL(r)   ((r)->delFlag == 1)
 #define RSMA_INFO_SET_DEL(r)  ((r)->delFlag = 1)
 #define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i])
-#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i])
 #define RSMA_INFO_ITEM(r, i)  (&(r)->items[i])

 enum {
@ -214,11 +219,11 @@ static FORCE_INLINE void tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
 int32_t smaPreClose(SSma *pSma);

 // rsma
-void   *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
+void   *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback);
 int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
 int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type);
-// int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
+int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
 int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback);
 void    tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName);

--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@ -43,9 +43,9 @@ extern "C" {

 typedef struct STqOffsetStore STqOffsetStore;

-// tqPush
 #define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1)
-#define STREAM_EXEC_TASK_STATUS_CHECK_ID     (-2)
+#define STREAM_EXEC_START_ALL_TASKS_ID     (-2)
+#define STREAM_EXEC_RESTART_ALL_TASKS_ID   (-3)
 #define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)

 // tqExec
@ -156,9 +156,6 @@ char*   tqOffsetBuildFName(const char* path, int32_t fVer);
 int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);

 // tqStream
-int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
-int32_t tqScanWal(STQ* pTq);
-int32_t tqStartStreamTask(STQ* pTq);
 int32_t tqResetStreamTaskStatus(STQ* pTq);
 int32_t tqStopStreamTasks(STQ* pTq);

--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@ -209,6 +209,7 @@ int32_t tsdbBegin(STsdb* pTsdb);
 // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);
 int32_t tsdbCacheCommit(STsdb* pTsdb);
 int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
+int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync);
 // int32_t tsdbFinishCommit(STsdb* pTsdb);
 // int32_t tsdbRollbackCommit(STsdb* pTsdb);
 int     tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
@ -231,7 +232,12 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
-int32_t tqLaunchStreamTaskAsync(STQ* pTq);
+
+int32_t tqStartStreamTaskAsync(STQ* pTq, bool restart);
+int32_t tqRestartStreamTasks(STQ* pTq);
+int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
+int32_t tqScanWal(STQ* pTq);
+int32_t tqStartStreamTasks(STQ* pTq);

 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
@ -274,7 +280,7 @@ int32_t smaPrepareAsyncCommit(SSma* pSma);
 int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo);
 int32_t smaFinishCommit(SSma* pSma);
 int32_t smaPostCommit(SSma* pSma);
-int32_t smaDoRetention(SSma* pSma, int64_t now);
+int32_t smaRetention(SSma* pSma, int64_t now);

 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
--- a/source/dnode/vnode/src/sma/smaCommit.c
+++ b/source/dnode/vnode/src/sma/smaCommit.c
@ -156,10 +156,10 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) {
  nLoops = 0;
  while (1) {
    if (atomic_load_32(&pRSmaStat->nFetchAll) <= 0) {
-      smaDebug("vgId:%d, rsma commit:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit);
+      smaDebug("vgId:%d, rsma commit, type:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit);
      break;
    } else {
-      smaDebug("vgId:%d, rsma commit%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit);
+      smaDebug("vgId:%d, rsma commit, type:%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit);
    }
    TD_SMA_LOOPS_CHECK(nLoops, 1000);
  }
@ -169,22 +169,24 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) {
   *  1) This is high cost task and should not put in asyncPreCommit originally.
   *  2) But, if put in asyncCommit, would trigger taskInfo cloning frequently.
   */
-  smaInfo("vgId:%d, rsma commit:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit,
+  smaInfo("vgId:%d, rsma commit, type:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit,
          (void *)taosGetSelfPthreadId());
  nLoops = 0;
  while (atomic_load_64(&pRSmaStat->nBufItems) > 0) {
    TD_SMA_LOOPS_CHECK(nLoops, 1000);
  }
+  smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());

  if (!isCommit) goto _exit;

-  // code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat));
+  code = atomic_load_32(&pRSmaStat->execStat);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat));
  TSDB_CHECK_CODE(code, lino, _exit);

  smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());

-  smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
-
  // all rsma results are written completely
  STsdb *pTsdb = NULL;
  if ((pTsdb = VND_RSMA1(pSma->pVnode))) {
--- a/source/dnode/vnode/src/sma/smaEnv.c
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@ -179,7 +179,7 @@ static void tRSmaInfoHashFreeNode(void *data) {
    if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) {
      taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES);
    }
-    tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true);
+    tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo);
  }
 }

@ -209,6 +209,12 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
      pRSmaStat->pSma = (SSma *)pSma;
      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);
      tsem_init(&pRSmaStat->notEmpty, 0, 0);
+      if (!(pRSmaStat->blocks = taosArrayInit(1, sizeof(SSDataBlock)))) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
+      SSDataBlock datablock = {.info.type = STREAM_CHECKPOINT};
+      taosArrayPush(pRSmaStat->blocks, &datablock);

      // init smaMgmt
      smaInit();
@ -290,6 +296,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {

    // step 5: free pStat
    tsem_destroy(&(pStat->notEmpty));
+    taosArrayDestroy(pStat->blocks);
    taosMemoryFreeClear(pStat);
  }
 }
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@ -15,12 +15,14 @@

 #include "sma.h"
 #include "tq.h"
+#include "tstream.h"

 #define RSMA_QTASKEXEC_SMOOTH_SIZE (100)     // cnt
 #define RSMA_SUBMIT_BATCH_SIZE     (1024)    // cnt
 #define RSMA_FETCH_DELAY_MAX       (120000)  // ms
 #define RSMA_FETCH_ACTIVE_MAX      (1000)    // ms
 #define RSMA_FETCH_INTERVAL        (5000)    // ms
+#define RSMA_TASK_FLAG             "rsma"

 #define RSMA_NEED_FETCH(r) (RSMA_INFO_ITEM((r), 0)->fetchLevel || RSMA_INFO_ITEM((r), 1)->fetchLevel)

@ -42,8 +44,8 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid);
 static void       tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 static void       tdFreeRSmaSubmitItems(SArray *pItems);
 static int32_t    tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo);
-static int32_t    tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
-                                            int64_t suid);
+static int32_t    tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo,
+                                            int32_t execType, int8_t *streamFlushed);
 static void       tdRSmaFetchTrigger(void *param, void *tmrId);
 static void       tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level);
 static int32_t    tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
@ -72,33 +74,32 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l
 *
 * @param pSma
 * @param pInfo
- * @param isDeepFree Only stop tmrId and free pTSchema for deep free
 * @return void*
 */
-void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
+void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
  if (pInfo) {
    for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
      SRSmaInfoItem *pItem = &pInfo->items[i];

-      if (isDeepFree && pItem->tmrId) {
+      if (pItem->tmrId) {
        smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId,
                 pInfo->suid, i + 1);
        taosTmrStopA(&pItem->tmrId);
      }

-      if (isDeepFree && pItem->pStreamState) {
+      if (pItem->pStreamState) {
        streamStateClose(pItem->pStreamState, false);
      }

-      if (isDeepFree && pInfo->taskInfo[i]) {
+      if (pItem->pStreamTask) {
+        tFreeStreamTask(pItem->pStreamTask);
+      }
+      taosArrayDestroy(pItem->pResList);
      tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1);
    }
-    }
-    if (isDeepFree) {
-      taosMemoryFreeClear(pInfo->pTSchema);
-    }

-    if (isDeepFree) {
+    taosMemoryFreeClear(pInfo->pTSchema);
+
    if (pInfo->queue) {
      taosCloseQueue(pInfo->queue);
      pInfo->queue = NULL;
@ -107,7 +108,6 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
      taosFreeQall(pInfo->qall);
      pInfo->qall = NULL;
    }
-    }

    taosMemoryFree(pInfo);
  }
@ -135,7 +135,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids,
    return TSDB_CODE_FAILED;
  }

-  if (!taosArrayGetSize(tbUids)) {
+  int32_t nTables = taosArrayGetSize(tbUids);
+
+  if (0 == nTables) {
    smaDebug("vgId:%d, no need to update tbUidList for suid:%" PRIi64 " since Empty tbUids", SMA_VID(pSma), *suid);
    return TSDB_CODE_SUCCESS;
  }
@ -156,8 +158,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids,
                 terrstr());
        return TSDB_CODE_FAILED;
      }
-      smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d",
-               SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i);
+      smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p. suid:%" PRIi64 " uid:%" PRIi64
+               "nTables:%d level %d",
+               SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)TARRAY_GET_ELEM(tbUids, 0), nTables, i);
    }
  }

@ -174,8 +177,8 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) {
    return TSDB_CODE_FAILED;
  }

-  void *pIter = taosHashIterate(pStore->uidHash, NULL);
-  while (pIter) {
+  void *pIter = NULL;
+  while ((pIter = taosHashIterate(pStore->uidHash, pIter))) {
    tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
    SArray   *pTbUids = *(SArray **)pIter;

@ -183,8 +186,6 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) {
      taosHashCancelIterate(pStore->uidHash, pIter);
      return TSDB_CODE_FAILED;
    }
-
-    pIter = taosHashIterate(pStore->uidHash, pIter);
  }
  return TSDB_CODE_SUCCESS;
 }
@ -232,9 +233,32 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
  return TSDB_CODE_SUCCESS;
 }

+static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTaskId *pId) {
+  STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
+  taosRLockLatch(&pMeta->lock);
+  SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
+  if (ppTask && *ppTask) {
+    pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer;
+    pItem->fetchResultVer = (*ppTask)->info.triggerParam;
+  }
+  taosRUnLockLatch(&pMeta->lock);
+}
+
+static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskId) {
+  streamMetaUnregisterTask(pMeta, streamId, taskId);
+  taosWLockLatch(&pMeta->lock);
+  int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
+  if (streamMetaCommit(pMeta) < 0) {
+    // persist to disk
+  }
+  taosWUnLockLatch(&pMeta->lock);
+  smaDebug("vgId:%d, rsma task:%" PRIi64 ",%d dropped, remain tasks:%d", pMeta->vgId, streamId, taskId, numOfTasks);
+}
+
 static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
                                       int8_t idx) {
  if ((param->qmsgLen > 0) && param->qmsg[idx]) {
+    SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
    SRetention    *pRetention = SMA_RETENTION(pSma);
    STsdbCfg      *pTsdbCfg = SMA_TSDB_CFG(pSma);
    SVnode        *pVnode = pSma->pVnode;
@ -254,25 +278,46 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
      taosMemoryFree(s);
    }

-    SStreamTask task = {.id.taskId = 0, .id.streamId = 0};  // TODO: assign value
-    task.pMeta = pVnode->pTq->pStreamMeta;
-    pStreamState = streamStateOpen(taskInfDir, &task, true, -1, -1);
+    SStreamTask *pStreamTask = taosMemoryCalloc(1, sizeof(*pStreamTask));
+    if (!pStreamTask) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+    pItem->pStreamTask = pStreamTask;
+    pStreamTask->id.taskId = 0;
+    pStreamTask->id.streamId = pRSmaInfo->suid + idx;
+    pStreamTask->chkInfo.startTs = taosGetTimestampMs();
+    pStreamTask->pMeta = pVnode->pTq->pStreamMeta;
+    pStreamTask->exec.qmsg = taosMemoryMalloc(strlen(RSMA_TASK_FLAG) + 1);
+    sprintf(pStreamTask->exec.qmsg, "%s", RSMA_TASK_FLAG);
+    pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta);
+    tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
+    pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1);
    if (!pStreamState) {
      terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
      return TSDB_CODE_FAILED;
    }
+    pItem->pStreamState = pStreamState;

-    SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState};
+    tdRSmaTaskRemove(pStreamTask->pMeta, pStreamTask->id.streamId, pStreamTask->id.taskId);
+
+    SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .skipRollup = 1, .pStateBackend = pStreamState};
    initStorageAPI(&handle.api);
-
    pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode), 0);
    if (!pRSmaInfo->taskInfo[idx]) {
      terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
      return TSDB_CODE_FAILED;
    }
-    SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
-    pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE;  // fetch the data when reboot
-    pItem->pStreamState = pStreamState;
+
+    if (!(pItem->pResList = taosArrayInit(1, POINTER_BYTES))) {
+      return TSDB_CODE_FAILED;
+    }
+
+    if (pItem->fetchResultVer < pItem->submitReqVer) {
+      // fetch the data when reboot
+      pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE;
+    }
+
    if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
      int64_t msInterval =
          convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND);
@ -291,10 +336,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat

    taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId);

-    smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64
+    smaInfo("vgId:%d, open rsma task:%p table:%" PRIi64 " level:%" PRIi8 ", checkpointId:%" PRIi64
+            ", submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64 ", maxdelay:%" PRIi64 " watermark:%" PRIi64
            ", finally maxdelay:%" PRIi32,
-            TD_VID(pVnode), pItem, pRSmaInfo->suid, (int8_t)(idx + 1), param->maxdelay[idx], param->watermark[idx],
-            pItem->maxDelay);
+            TD_VID(pVnode), pItem->pStreamTask, pRSmaInfo->suid, (int8_t)(idx + 1), pStreamTask->chkInfo.checkpointId,
+            pItem->submitReqVer, pItem->fetchResultVer, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay);
  }
  return TSDB_CODE_SUCCESS;
 }
@ -362,7 +408,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con

  return TSDB_CODE_SUCCESS;
 _err:
-  tdFreeRSmaInfo(pSma, pRSmaInfo, true);
+  tdFreeRSmaInfo(pSma, pRSmaInfo);
  return TSDB_CODE_FAILED;
 }

@ -498,11 +544,10 @@ static void tdUidStoreDestory(STbUidStore *pStore) {
    if (pStore->uidHash) {
      if (pStore->tbUids) {
        // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys.
-        void *pIter = taosHashIterate(pStore->uidHash, NULL);
-        while (pIter) {
+        void *pIter = NULL;
+        while ((pIter = taosHashIterate(pStore->uidHash, pIter))) {
          SArray *arr = *(SArray **)pIter;
          taosArrayDestroy(arr);
-          pIter = taosHashIterate(pStore->uidHash, pIter);
        }
      }
      taosHashCleanup(pStore->uidHash);
@ -562,7 +607,7 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) {
 * @param now
 * @return int32_t
 */
-int32_t smaDoRetention(SSma *pSma, int64_t now) {
+int32_t smaRetention(SSma *pSma, int64_t now) {
  int32_t code = TSDB_CODE_SUCCESS;
  if (!VND_IS_RSMA(pSma->pVnode)) {
    return code;
@ -570,8 +615,8 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) {

  for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
    if (pSma->pRSmaTsdb[i]) {
-      // code = tsdbDoRetention(pSma->pRSmaTsdb[i], now);
-      // if (code) goto _end;
+      code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
+      if (code) goto _end;
    }
  }

@ -579,17 +624,14 @@ _end:
  return code;
 }

-static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
-                                         int64_t suid) {
+static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo,
+                                         int32_t execType, int8_t *streamFlushed) {
  int32_t      code = 0;
  int32_t      lino = 0;
  SSDataBlock *output = NULL;
-
-  SArray *pResList = taosArrayInit(1, POINTER_BYTES);
-  if (pResList == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
+  SArray      *pResList = pItem->pResList;
+  STSchema    *pTSchema = pInfo->pTSchema;
+  int64_t      suid = pInfo->suid;

  while (1) {
    uint64_t ts;
@ -604,32 +646,46 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
    if (taosArrayGetSize(pResList) == 0) {
      break;
    }
-#if 0
-    char flag[10] = {0};
-    snprintf(flag, 10, "level %" PRIi8, pItem->level);
-    blockDebugShowDataBlocks(pResList, flag);
-#endif
+
    for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
      output = taosArrayGetP(pResList, i);
+      if (output->info.type == STREAM_CHECKPOINT) {
+        if (streamFlushed) *streamFlushed = 1;
+        continue;
+      }
      smaDebug("vgId:%d, result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRIi64, SMA_VID(pSma),
               output->info.id.uid, output->info.id.groupId, output->info.rows);

      STsdb       *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
      SSubmitReq2 *pReq = NULL;

-      // TODO: the schema update should be handled later(TD-17965)
      if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) {
        code = terrno ? terrno : TSDB_CODE_RSMA_RESULT;
        TSDB_CHECK_CODE(code, lino, _exit);
      }

+      // reset the output version to handle reboot
+      if (STREAM_GET_ALL == execType && output->info.version == 0) {
+        // the submitReqVer keeps unchanged since tdExecuteRSmaImpl and tdRSmaFetchAllResult are executed synchronously
+        output->info.version = pItem->submitReqVer;
+      }
+
      if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) {
-        code = terrno ? terrno : TSDB_CODE_RSMA_RESULT;
+        if (terrno == TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE) {
+          // TODO: reconfigure SSubmitReq2
+        } else {
+          if (terrno == 0) terrno = TSDB_CODE_RSMA_RESULT;
+          code = terrno;
+        }
        tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
        taosMemoryFree(pReq);
        TSDB_CHECK_CODE(code, lino, _exit);
      }

+      if (STREAM_GET_ALL == execType) {
+        atomic_store_64(&pItem->fetchResultVer, output->info.version);
+      }
+
      smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64,
               SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version);

@ -648,7 +704,6 @@ _exit:
  } else {
    smaDebug("vgId:%d, %s succeed, suid:%" PRIi64 ", level:%" PRIi8, SMA_VID(pSma), __func__, suid, pItem->level);
  }
-  taosArrayDestroy(pResList);
  qCleanExecTaskBlockBuf(taskInfo);
  return code;
 }
@ -745,6 +800,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize,
                                 ERsmaExecType type, int8_t level) {
  int32_t        idx = level - 1;
  void          *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx);
+  SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx);

  if (!qTaskInfo) {
    smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level,
@ -772,10 +828,14 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize,
    return TSDB_CODE_FAILED;
  }

-  SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx);
-  tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid);
+  if (STREAM_INPUT__MERGED_SUBMIT == inputType) {
+    SPackedData *packData = POINTER_SHIFT(pMsg, sizeof(SPackedData) * (msgSize - 1));
+    atomic_store_64(&pItem->submitReqVer, packData->ver);
+  }

-  return TSDB_CODE_SUCCESS;
+  terrno = tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo, STREAM_NORMAL, NULL);
+
+  return terrno ? TSDB_CODE_FAILED : TDB_CODE_SUCCESS;
 }

 /**
@ -881,7 +941,12 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg,
  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
  if (!pEnv) {
    // only applicable when rsma env exists
-    return TSDB_CODE_SUCCESS;
+    return TDB_CODE_SUCCESS;
+  }
+
+  if ((terrno = atomic_load_32(&SMA_RSMA_STAT(pSma)->execStat))) {
+    smaError("vgId:%d, failed to process rsma submit since invalid exec code: %s", SMA_VID(pSma), terrstr());
+    goto _err;
  }

  STbUidStore uidStore = {0};
@ -913,7 +978,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg,
  return TSDB_CODE_SUCCESS;
 _err:
  tdUidStoreDestory(&uidStore);
-  return TSDB_CODE_FAILED;
+  return terrno;
 }

 /**
@ -988,7 +1053,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
        code = terrno;
        TSDB_CHECK_CODE(code, lino, _exit);
      }
-
+#if 0
      // reload all ctbUids for suid
      uidStore.suid = suid;
      if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) {
@ -1002,7 +1067,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
      }

      taosArrayClear(uidStore.tbUids);
-
+#endif
      smaDebug("vgId:%d, rsma restore env success for %" PRIi64, TD_VID(pVnode), suid);
    }
  }
@ -1050,39 +1115,141 @@ _err:

  return code;
 }
-#if 0
+
 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
  int32_t code = 0;
  int32_t lino = 0;
+  int32_t nTaskInfo = 0;
  SSma   *pSma = pRSmaStat->pSma;
  SVnode *pVnode = pSma->pVnode;
-  SRSmaFS fs = {0};

  if (taosHashGetSize(pInfoHash) <= 0) {
    return TSDB_CODE_SUCCESS;
  }

+  // stream state: trigger checkpoint
+  do {
    void *infoHash = NULL;
    while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
      SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
+      if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+        continue;
+      }
+      for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+        if (pRSmaInfo->taskInfo[i]) {
+          code = qSetSMAInput(pRSmaInfo->taskInfo[i], pRSmaStat->blocks, 1, STREAM_INPUT__CHECKPOINT);
+          if (code) {
+            taosHashCancelIterate(pInfoHash, infoHash);
+            TSDB_CHECK_CODE(code, lino, _exit);
+          }
+          pRSmaInfo->items[i].streamFlushed = 0;
+          ++nTaskInfo;
+        }
+      }
+    }
+  } while (0);

+  // stream state: wait checkpoint ready in async mode
+  do {
+    int32_t nStreamFlushed = 0;
+    int32_t nSleep = 0;
+    void   *infoHash = NULL;
+    while (true) {
+      while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
+        SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
+        if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+          continue;
+        }
+        for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+          if (pRSmaInfo->taskInfo[i] && (0 == pRSmaInfo->items[i].streamFlushed)) {
+            int8_t streamFlushed = 0;
+            code = tdRSmaExecAndSubmitResult(pSma, pRSmaInfo->taskInfo[i], &pRSmaInfo->items[i], pRSmaInfo,
+                                             STREAM_CHECKPOINT, &streamFlushed);
+            if (code) {
+              taosHashCancelIterate(pInfoHash, infoHash);
+              TSDB_CHECK_CODE(code, lino, _exit);
+            }
+
+            if (streamFlushed) {
+              pRSmaInfo->items[i].streamFlushed = 1;
+              if (++nStreamFlushed >= nTaskInfo) {
+                smaInfo("vgId:%d, rsma commit, checkpoint ready, %d us consumed, received/total: %d/%d", TD_VID(pVnode),
+                        nSleep * 10, nStreamFlushed, nTaskInfo);
+                taosHashCancelIterate(pInfoHash, infoHash);
+                goto _checkpoint;
+              }
+            }
+          }
+        }
+      }
+      taosUsleep(10);
+      ++nSleep;
+      smaDebug("vgId:%d, rsma commit, wait for checkpoint ready, %d us elapsed, received/total: %d/%d", TD_VID(pVnode),
+               nSleep * 10, nStreamFlushed, nTaskInfo);
+    }
+  } while (0);
+
+_checkpoint:
+  // stream state: build checkpoint in backend
+  do {
+    SStreamMeta *pMeta = NULL;
+    int64_t      checkpointId = taosGetTimestampNs();
+    bool         checkpointBuilt = false;
+    void        *infoHash = NULL;
+    while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
+      SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
      if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
        continue;
      }

      for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
        SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i);
-      if (pItem && pItem->pStreamState) {
-        if (streamStateCommit(pItem->pStreamState) < 0) {
-          code = TSDB_CODE_RSMA_STREAM_STATE_COMMIT;
+        if (pItem && pItem->pStreamTask) {
+          SStreamTask *pTask = pItem->pStreamTask;
+          atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1);
+          pTask->checkpointingId = checkpointId;
+          pTask->chkInfo.checkpointId = pTask->checkpointingId;
+          pTask->chkInfo.checkpointVer = pItem->submitReqVer;
+          pTask->info.triggerParam = pItem->fetchResultVer;
+
+          if (!checkpointBuilt) {
+            // the stream states share one checkpoint
+            code = streamTaskBuildCheckpoint(pTask);
+            if (code) {
+              taosHashCancelIterate(pInfoHash, infoHash);
              TSDB_CHECK_CODE(code, lino, _exit);
            }
-        smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 ", level %d", TD_VID(pVnode),
-                 pRSmaInfo->suid, i + 1);
-      }
-    }
+            pMeta = pTask->pMeta;
+            checkpointBuilt = true;
          }

+          taosWLockLatch(&pMeta->lock);
+          if (streamMetaSaveTask(pMeta, pTask)) {
+            taosWUnLockLatch(&pMeta->lock);
+            code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY;
+            taosHashCancelIterate(pInfoHash, infoHash);
+            TSDB_CHECK_CODE(code, lino, _exit);
+          }
+          taosWUnLockLatch(&pMeta->lock);
+          smaDebug("vgId:%d, rsma commit, succeed to commit task:%p, submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64
+                   ", table:%" PRIi64 ", level:%d",
+                   TD_VID(pVnode), pTask, pItem->submitReqVer, pItem->fetchResultVer, pRSmaInfo->suid, i + 1);
+        }
+      }
+    }
+    if (pMeta) {
+      taosWLockLatch(&pMeta->lock);
+      if (streamMetaCommit(pMeta)) {
+        taosWUnLockLatch(&pMeta->lock);
+        code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY;
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
+      taosWUnLockLatch(&pMeta->lock);
+    }
+    if (checkpointBuilt) {
+      smaInfo("vgId:%d, rsma commit, succeed to commit checkpoint:%" PRIi64, TD_VID(pVnode), checkpointId);
+    }
+  } while (0);
 _exit:
  if (code) {
    smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
@ -1091,7 +1258,7 @@ _exit:
  terrno = code;
  return code;
 }
-#endif
+
 /**
 * @brief trigger to get rsma result in async mode
 *
@ -1165,14 +1332,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
      smaDebug("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64 " since stat is active",
               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
      // async process
-      pItem->fetchLevel = pItem->level;
-#if 0
-      // debugging codes
-      SRSmaInfo     *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid);
-      SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1);
-      make sure(qItem->level == pItem->level);
-      make sure(qItem->fetchLevel == pItem->fetchLevel);
-#endif
+      atomic_store_8(&pItem->fetchLevel, 1);
+
      if (atomic_load_8(&pRSmaInfo->assigned) == 0) {
        tsem_post(&(pStat->notEmpty));
      }
@ -1217,15 +1378,15 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) {
  SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
  for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) {
    SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1);
-    if (pItem->fetchLevel) {
-      pItem->fetchLevel = 0;
+
+    if (1 == atomic_val_compare_exchange_8(&pItem->fetchLevel, 1, 0)) {
      qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, i - 1);
      if (!taskInfo) {
        continue;
      }

      if ((++pItem->nScanned * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) {
-        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16 " maxDelay:%d, fetch executed",
+        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch executed",
                 SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay);
      } else {
        int64_t curMs = taosGetTimestampMs();
@ -1245,14 +1406,15 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) {
      if ((terrno = qSetSMAInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) {
        goto _err;
      }
-      if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) {
+      if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo, STREAM_GET_ALL, NULL) < 0) {
+        atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno);
        goto _err;
      }

-      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16 " maxDelay:%d, fetch finished",
+      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch finished",
               SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay);
    } else {
-      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16
+      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32
               " maxDelay:%d, fetch not executed as fetch level is %" PRIi8,
               SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay, pItem->fetchLevel);
    }
@ -1275,6 +1437,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
                              .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))};

      if (!taosArrayPush(pSubmitArr, &packData)) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
        tdFreeRSmaSubmitItems(pSubmitArr);
        goto _err;
      }
@ -1294,6 +1457,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
  }
  return TSDB_CODE_SUCCESS;
 _err:
+  atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno);
  smaError("vgId:%d, batch exec for suid:%" PRIi64 " execType:%d size:%d failed since %s", SMA_VID(pSma), pInfo->suid,
           type, (int32_t)taosArrayGetSize(pSubmitArr), terrstr());
  tdFreeRSmaSubmitItems(pSubmitArr);
@ -1369,15 +1533,11 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {

                  if (ASSERTS(oldVal >= 0, "oldVal of nFetchAll: %d < 0", oldVal)) {
                    code = TSDB_CODE_APP_ERROR;
+                    taosHashCancelIterate(infoHash, pIter);
                    TSDB_CHECK_CODE(code, lino, _exit);
                  }

-                  int8_t curStat = atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat));
-                  if (curStat == 1) {
-                    smaDebug("vgId:%d, fetch all not exec as commit stat is %" PRIi8, SMA_VID(pSma), curStat);
-                  } else {
                  tdRSmaFetchAllResult(pSma, pInfo);
-                  }

                  if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) {
                    atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@ -1064,7 +1064,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
  return code;
 }

-static void doStartStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) {
+static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) {
  const char* id = pTask->id.idStr;
  int64_t     nextProcessedVer = pStreamTask->hTaskInfo.haltVer;

@ -1105,7 +1105,11 @@ static void doStartStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq)
  }
 }

-// this function should be executed by only one thread
+static void ddxx() {
+
+}
+
+// this function should be executed by only one thread, so we set an sentinel to protect this function
 int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
  SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont;
  SStreamMeta*           pMeta = pTq->pStreamMeta;
@ -1134,6 +1138,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
    }
  }

+  // let's decide which step should be executed now
  if (pTask->execInfo.step1Start == 0) {
    ASSERT(pTask->status.pauseAllowed == false);
    int64_t ts = taosGetTimestampMs();
@ -1147,7 +1152,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
    }
  } else {
    if (pTask->execInfo.step2Start == 0) {
-      tqDebug("s-task:%s resume from paused, original step1 startTs:%" PRId64, id, pTask->execInfo.step1Start);
+      tqDebug("s-task:%s continue exec scan-history(step1), original step1 startTs:%" PRId64 ", already elapsed:%.2fs",
+              id, pTask->execInfo.step1Start, pTask->execInfo.step1El);
    } else {
      tqDebug("s-task:%s already in step2, no need to scan-history data, step2 starTs:%"PRId64, id, pTask->execInfo.step2Start);
      atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
@ -1167,20 +1173,37 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
    return 0;
  }

-  streamScanHistoryData(pTask);
+  int64_t st = taosGetTimestampMs();
+  SScanhistoryDataInfo retInfo = streamScanHistoryData(pTask, st);

-  double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
-  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE) {
+  double el = (taosGetTimestampMs() - st) / 1000.0;
+  pTask->execInfo.step1El += el;
+
+  if (retInfo.ret == TASK_SCANHISTORY_QUIT || retInfo.ret == TASK_SCANHISTORY_REXEC) {
    int8_t status = streamTaskSetSchedStatusInactive(pTask);
-    tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status);
-
    atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
+
+    if (retInfo.ret == TASK_SCANHISTORY_REXEC) {
+      streamReExecScanHistoryFuture(pTask, retInfo.idleTime);
+    } else {
+      char*       p = NULL;
+      ETaskStatus s = streamTaskGetStatus(pTask, &p);
+
+      if (s == TASK_STATUS__PAUSE) {
+        tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr,
+                el, pTask->execInfo.step1El, status);
+      } else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) {
+        tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr, p,
+                pTask->execInfo.step1El);
+      }
+    }
+
    streamMetaReleaseTask(pMeta, pTask);
    return 0;
  }

  // the following procedure should be executed, no matter status is stop/pause or not
-  tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, el);
+  tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El);

  if (pTask->info.fillHistory) {
    SStreamTask*   pStreamTask = NULL;
@ -1203,21 +1226,18 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {

    code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
    if (code == TSDB_CODE_SUCCESS) {
-      doStartStep2(pTask, pStreamTask, pTq);
+      doStartFillhistoryStep2(pTask, pStreamTask, pTq);
    } else {
      tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr);
    }

    streamMetaReleaseTask(pMeta, pStreamTask);
-
  } else {
    STimeWindow* pWindow = &pTask->dataRange.window;
    ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));

-    // Not update the fill-history time window until the state transfer is completed if the related fill-history task
-    // exists.
-    tqDebug(
-        "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, startVer:%" PRId64
+    // Not update the fill-history time window until the state transfer is completed.
+    tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64
            ", window:%" PRId64 " - %" PRId64,
            id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey);

@ -1297,14 +1317,15 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
  int32_t taskId = pReq->taskId;
  int32_t vgId = TD_VID(pTq->pVnode);

-  if (taskId == STREAM_EXEC_TASK_STATUS_CHECK_ID) {
-    tqStartStreamTask(pTq);
-    return 0;
-  }
-
  if (taskId == STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID) {  // all tasks are extracted submit data from the wal
    tqScanWal(pTq);
    return 0;
+  } else if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) {
+    tqStartStreamTasks(pTq);
+    return 0;
+  } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) {
+    tqRestartStreamTasks(pTq);
+    return 0;
  }

  SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId);
@ -1492,6 +1513,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
      streamSchedExec(pTask);
    }
  } else if (status == TASK_STATUS__UNINIT) {
+    // todo: fill-history task init ?
    if (pTask->info.fillHistory == 0) {
      EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
      streamTaskHandleEvent(pTask->status.pSM, event);
@ -1890,7 +1912,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
  int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
  int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks);

-  pMeta->startInfo.startAllTasksFlag = 1;
+  pMeta->startInfo.tasksWillRestart = 1;

  if (updateTasks < numOfTasks) {
    tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId,
@ -1899,45 +1921,11 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
  } else {
    if (!pTq->pVnode->restored) {
      tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId);
-      pMeta->startInfo.startAllTasksFlag = 0;
+      pMeta->startInfo.tasksWillRestart = 0;
      streamMetaWUnLock(pMeta);
    } else {
-      tqInfo("vgId:%d tasks are all updated and stopped, restart them", vgId);
-      terrno = 0;
-
-      streamMetaWUnLock(pMeta);
-
-      while (streamMetaTaskInTimer(pMeta)) {
-        tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
-        taosMsleep(100);
-      }
-
-      streamMetaWLock(pMeta);
-
-      int32_t code = streamMetaReopen(pMeta);
-      if (code != 0) {
-        tqError("vgId:%d failed to reopen stream meta", vgId);
-        streamMetaWUnLock(pMeta);
-        taosArrayDestroy(req.pNodeList);
-        return -1;
-      }
-
-      if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) {
-        tqError("vgId:%d failed to load stream tasks", vgId);
-        streamMetaWUnLock(pMeta);
-        taosArrayDestroy(req.pNodeList);
-        return -1;
-      }
-
-      if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) {
-        tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
-        tqResetStreamTaskStatus(pTq);
-        tqLaunchStreamTaskAsync(pTq);
-      } else {
-        tqInfo("vgId:%d, follower node not start stream tasks", vgId);
-      }
-
      streamMetaWUnLock(pMeta);
+      tqStartStreamTaskAsync(pTq, true);
    }
  }

--- a/source/dnode/vnode/src/tq/tqStreamTask.c
+++ b/source/dnode/vnode/src/tq/tqStreamTask.c
@ -22,6 +22,8 @@
 static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle);
 static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId);
 static bool    handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver);
+static bool    taskReadyForDataFromWal(SStreamTask* pTask);
+static bool    doPutDataIntoInputQFromWal(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems);

 // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks.
 int32_t tqScanWal(STQ* pTq) {
@ -58,7 +60,7 @@ int32_t tqScanWal(STQ* pTq) {
  return 0;
 }

-int32_t tqStartStreamTask(STQ* pTq) {
+int32_t tqStartStreamTasks(STQ* pTq) {
  int32_t      code = TSDB_CODE_SUCCESS;
  int32_t      vgId = TD_VID(pTq->pVnode);
  SStreamMeta* pMeta = pTq->pStreamMeta;
@ -73,6 +75,7 @@ int32_t tqStartStreamTask(STQ* pTq) {
  streamMetaWLock(pMeta);
  pTaskList = taosArrayDup(pMeta->pTaskList, NULL);
  taosHashClear(pMeta->startInfo.pReadyTaskSet);
+  taosHashClear(pMeta->startInfo.pFailedTaskSet);
  pMeta->startInfo.startTs = taosGetTimestampMs();
  streamMetaWUnLock(pMeta);

@ -97,7 +100,7 @@ int32_t tqStartStreamTask(STQ* pTq) {
        streamLaunchFillHistoryTask(pTask);
      }

-      streamMetaUpdateTaskReadyInfo(pTask);
+      streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true);
      streamMetaReleaseTask(pMeta, pTask);
      continue;
    }
@ -115,7 +118,67 @@ int32_t tqStartStreamTask(STQ* pTq) {
  return code;
 }

-int32_t tqLaunchStreamTaskAsync(STQ* pTq) {
+int32_t tqRestartStreamTasks(STQ* pTq) {
+  SStreamMeta* pMeta = pTq->pStreamMeta;
+  int32_t      vgId = pMeta->vgId;
+  int32_t      code = 0;
+  int64_t      st = taosGetTimestampMs();
+
+  while(1) {
+    int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1);
+    if (startVal == 0) {
+      break;
+    }
+
+    tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId);
+    taosMsleep(500);
+  }
+
+  terrno = 0;
+  tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId,
+         pMeta->updateInfo.transId);
+
+  while (streamMetaTaskInTimer(pMeta)) {
+    tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
+    taosMsleep(100);
+  }
+
+  streamMetaWLock(pMeta);
+
+  code = streamMetaReopen(pMeta);
+  if (code != TSDB_CODE_SUCCESS) {
+    tqError("vgId:%d failed to reopen stream meta", vgId);
+    streamMetaWUnLock(pMeta);
+    code = terrno;
+    return code;
+  }
+
+  int64_t el = taosGetTimestampMs() - st;
+
+  tqInfo("vgId:%d close&reload state elapsed time:%.3fms", vgId, el/1000.);
+
+  code = streamMetaLoadAllTasks(pTq->pStreamMeta);
+  if (code != TSDB_CODE_SUCCESS) {
+    tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno));
+    streamMetaWUnLock(pMeta);
+    code = terrno;
+    return code;
+  }
+
+  if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) {
+    tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
+    tqResetStreamTaskStatus(pTq);
+    tqStartStreamTasks(pTq);
+  } else {
+    tqInfo("vgId:%d, follower node not start stream tasks", vgId);
+  }
+
+  streamMetaWUnLock(pMeta);
+  code = terrno;
+  return code;
+}
+
+int32_t tqStartStreamTaskAsync(STQ* pTq, bool restart) {
  SStreamMeta* pMeta = pTq->pStreamMeta;
  int32_t      vgId = pMeta->vgId;

@ -132,10 +195,10 @@ int32_t tqLaunchStreamTaskAsync(STQ* pTq) {
    return -1;
  }

-  tqDebug("vgId:%d check %d stream task(s) status async", vgId, numOfTasks);
+  tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks);
  pRunReq->head.vgId = vgId;
  pRunReq->streamId = 0;
-  pRunReq->taskId = STREAM_EXEC_TASK_STATUS_CHECK_ID;
+  pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID;

  SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
  tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg);
@ -320,14 +383,13 @@ bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) {
  return false;
 }

-static bool taskReadyForDataFromWal(SStreamTask* pTask) {
+bool taskReadyForDataFromWal(SStreamTask* pTask) {
  // non-source or fill-history tasks don't need to response the WAL scan action.
  if ((pTask->info.taskLevel != TASK_LEVEL__SOURCE) || (pTask->status.downstreamReady == 0)) {
    return false;
  }

  // not in ready state, do not handle the data from wal
-//  int32_t status = pTask->status.taskStatus;
  char* p = NULL;
  int32_t status = streamTaskGetStatus(pTask, &p);
  if (streamTaskGetStatus(pTask, &p) != TASK_STATUS__READY) {
@ -359,7 +421,7 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
  return true;
 }

-static bool doPutDataIntoInputQFromWal(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems) {
+bool doPutDataIntoInputQFromWal(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems) {
  const char* id = pTask->id.idStr;
  int32_t     numOfNewItems = 0;

@ -449,7 +511,6 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
    int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX;

    taosThreadMutexLock(&pTask->lock);
-    tqDebug("s-task:%s lock", pTask->id.idStr);

    char* p = NULL;
    ETaskStatus status = streamTaskGetStatus(pTask, &p);
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@ -12,11 +12,11 @@
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
+#include "cos.h"
 #include "tsdb.h"
 #include "tsdbDataFileRW.h"
 #include "tsdbReadUtil.h"
 #include "vnd.h"
-#include "vndCos.h"

 #define ROCKS_BATCH_SIZE (4096)

@ -3099,7 +3099,7 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) {
  }
  */
  int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage;
-  code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, ppBlock);
+  code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock);
  if (code != TSDB_CODE_SUCCESS) {
    // taosMemoryFree(pBlock);
    // code = TSDB_CODE_OUT_OF_MEMORY;
--- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
@ -28,13 +28,18 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
  // bool    allNullRow = true;

  if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) {
+    uint64_t ts = 0;
+    SFirstLastRes* p;
+    col_id_t colId;
    for (int32_t i = 0; i < pReader->numOfCols; ++i) {
      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
-      SFirstLastRes*   p = (SFirstLastRes*)varDataVal(pRes[i]);
      int32_t          slotId = slotIds[i];
      SLastCol*        pColVal = (SLastCol*)taosArrayGet(pRow, i);
+      colId = pColVal->colVal.cid;
+      p = (SFirstLastRes*)varDataVal(pRes[i]);

      p->ts = pColVal->ts;
+      ts = p->ts;
      p->isNull = !COL_VAL_IS_VALUE(&pColVal->colVal);
      // allNullRow = p->isNull & allNullRow;

@ -55,6 +60,19 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
      varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE);
      colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false);
    }
+    for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) {
+      SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx);
+      if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
+        colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
+        continue;
+      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) {
+        if (!p->isNull) {
+          colDataSetVal(pCol, numOfRows, p->buf, false);
+        } else {
+          colDataSetNULL(pCol, numOfRows);
+        }
+      }
+    }

    // pBlock->info.rows += allNullRow ? 0 : 1;
    ++pBlock->info.rows;
@ -144,7 +162,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
  p->type = type;
  p->pVnode = pVnode;
  p->pTsdb = p->pVnode->pTsdb;
-  p->info.verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX};
+  p->info.verRange = (SVersionRange){.minVer = 0, .maxVer = INT64_MAX};
  p->info.suid = suid;
  p->numOfCols = numOfCols;
  p->pCidList = pCidList;
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@ -192,7 +192,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
      committer->ctx->tbid->uid = record->uid;

      if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) {
-        code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
+        code = tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid);
        TSDB_CHECK_CODE(code, lino, _exit);
        continue;
      }
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c
@ -14,15 +14,15 @@
 */

 #include "tsdbFS2.h"
+#include "cos.h"
 #include "tsdbUpgrade.h"
 #include "vnd.h"
-#include "vndCos.h"

 #define BLOCK_COMMIT_FACTOR 3

 extern int  vnodeScheduleTask(int (*execute)(void *), void *arg);
 extern int  vnodeScheduleTaskEx(int tpid, int (*execute)(void *), void *arg);
-extern void remove_file(const char *fname);
+extern void remove_file(const char *fname, bool last_level);

 #define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
 #define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
@ -532,7 +532,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
      if (taosIsDir(file->aname)) continue;

      if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL) {
-        remove_file(file->aname);
+        int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
+        remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1);
      }
    }

--- a/source/dnode/vnode/src/tsdb/tsdbFile2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c
@ -14,6 +14,7 @@
 */

 #include "tsdbFile2.h"
+#include "cos.h"

 // to_json
 static int32_t head_to_json(const STFile *file, cJSON *json);
@ -41,10 +42,20 @@ static const struct {
    [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
 };

-void remove_file(const char *fname) {
+void remove_file(const char *fname, bool last_level) {
  int32_t code = taosRemoveFile(fname);
  if (code) {
+    if (tsS3Enabled && last_level) {
+      const char *object_name = taosDirEntryBaseName((char *)fname);
+      long        s3_size = tsS3Enabled ? s3Size(object_name) : 0;
+      if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) {
+        s3DeleteObjects(&object_name, 1);
+      } else {
        tsdbError("file:%s remove failed", fname);
+      }
+    } else {
+      tsdbError("file:%s remove failed", fname);
+    }
  } else {
    tsdbInfo("file:%s is removed", fname);
  }
@ -224,6 +235,7 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
  fobj[0]->state = TSDB_FSTATE_LIVE;
  fobj[0]->ref = 1;
  tsdbTFileName(pTsdb, f, fobj[0]->fname);
+  fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
  return 0;
 }

@ -245,7 +257,7 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
  tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
    if (fobj->state == TSDB_FSTATE_DEAD) {
-      remove_file(fobj->fname);
+      remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
    }
    taosMemoryFree(fobj);
  }
@ -261,7 +273,7 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) {
  taosThreadMutexUnlock(&fobj->mutex);
  tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
-    remove_file(fobj->fname);
+    remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
    taosMemoryFree(fobj);
  }
  return 0;
--- a/source/dnode/vnode/src/tsdb/tsdbFile2.h
+++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h
@ -76,6 +76,7 @@ struct STFileObj {
  STFile        f[1];
  int32_t       state;
  int32_t       ref;
+  int32_t       nlevel;
  char          fname[TSDB_FILENAME_LEN];
 };

--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@ -846,14 +846,14 @@ static void tLDataIterPinSttBlock(SLDataIter* pIter, const char* id) {
  if (pInfo->blockData[0].sttBlockIndex == pIter->iSttBlk) {
    pInfo->blockData[0].pin = true;
    ASSERT(!pInfo->blockData[1].pin);
-    tsdbDebug("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id);
+    tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id);
    return;
  }

  if (pInfo->blockData[1].sttBlockIndex == pIter->iSttBlk) {
    pInfo->blockData[1].pin = true;
    ASSERT(!pInfo->blockData[0].pin);
-    tsdbDebug("pin stt-block, blockIndex:%d, stt-fileVer:%"PRId64" %s", pIter->iSttBlk, pIter->cid, id);
+    tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%"PRId64" %s", pIter->iSttBlk, pIter->cid, id);
    return;
  }

--- a/source/dnode/vnode/src/tsdb/tsdbOpen.c
+++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c
@ -89,6 +89,8 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
  return 0;

 _err:
+  tsdbCloseFS(&pTsdb->pFS);
+  taosThreadMutexDestroy(&pTsdb->mutex);
  taosMemoryFree(pTsdb);
  return -1;
 }
--- a/source/dnode/vnode/src/tsdb/tsdbRead2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c
@ -49,7 +49,7 @@ static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo
                                     STsdbReader* pReader);

 static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost);
-static STsdb*        getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr,
+static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
                                  int8_t* pLevel);
 static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level);
 static bool          hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
@ -384,7 +384,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void

  initReaderStatus(&pReader->status);

-  pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
+  pReader->pTsdb = getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level);
  pReader->info.suid = pCond->suid;
  pReader->info.order = pCond->order;

@ -565,9 +565,9 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN

    STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
    if (ASCENDING_TRAVERSE(pReader->info.order)) {
-      w.skey = pScanInfo->lastKey + step;
+      w.skey = pScanInfo->lastProcKey + step;
    } else {
-      w.ekey = pScanInfo->lastKey + step;
+      w.ekey = pScanInfo->lastProcKey + step;
    }

    if (isEmptyQueryTimeWindow(&w)) {
@ -607,14 +607,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN

  clearBrinBlockIter(&iter);

-  pBlockNum->numOfLastFiles = pReader->status.pCurrentFileset->lvlArr->size;
-  int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks;
+  pBlockNum->numOfSttFiles = pReader->status.pCurrentFileset->lvlArr->size;
+  int32_t total = pBlockNum->numOfSttFiles + pBlockNum->numOfBlocks;

  double el = (taosGetTimestampUs() - st) / 1000.0;
  tsdbDebug(
      "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
      "time:%.2f ms %s",
-      numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfLastFiles,
+      numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfSttFiles,
      sizeInDisk / 1000.0, el, pReader->idStr);

  pReader->cost.numOfBlocks += total;
@ -1200,13 +1200,12 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
  }
 }

-static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, SLastBlockReader* pLastBlockReader, int32_t order) {
+static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo, int32_t order) {
  bool ascScan = ASCENDING_TRAVERSE(order);
-  bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);

  int64_t key = 0;
-  if (bHasDataInLastBlock) {
-    int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+  if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
+    int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
    key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt);
  } else {
    key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey;
@ -1215,10 +1214,10 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, SLastBlockReader
  return key;
 }

-static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock,
-                                     SLastBlockReader* pLastBlockReader, int32_t order) {
+static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo,
+                                     int32_t order) {
  bool    ascScan = ASCENDING_TRAVERSE(order);
-  int64_t key = getBoarderKeyInFiles(pBlock, pLastBlockReader, order);
+  int64_t key = getBoarderKeyInFiles(pBlock, pScanInfo, order);

  return (ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts < key)) ||
         (!ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts > key));
@ -1302,10 +1301,9 @@ typedef struct {
 } SDataBlockToLoadInfo;

 static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo,
-                               STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader,
-                               STsdbReader* pReader) {
-  int32_t     neighborIndex = 0;
+                               STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) {
  SBrinRecord rec = {0};
+  int32_t     neighborIndex = 0;

  bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex,
                                                 pReader->info.order, &rec);
@ -1319,9 +1317,11 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo*
  pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count) || (pBlockInfo->record.count <= 0);
  pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order);

-  if (hasDataInLastBlock(pLastBlockReader)) {
-    int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
-    pInfo->overlapWithLastBlock = !(pBlockInfo->record.lastKey < tsLast || pBlockInfo->record.firstKey > tsLast);
+  ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT);
+  if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
+    int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey;
+    pInfo->overlapWithLastBlock =
+        !(pBlockInfo->record.lastKey < nextProcKeyInStt || pBlockInfo->record.firstKey > nextProcKeyInStt);
  }

  pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity;
@ -1336,9 +1336,9 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo*
 // 5. delete info should not overlap with current block data
 // 6. current block should not contain the duplicated ts
 static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo,
-                                TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) {
+                                TSDBKEY keyInBuf) {
  SDataBlockToLoadInfo info = {0};
-  getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader);
+  getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader);

  bool loadDataBlock =
      (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf ||
@ -1358,9 +1358,9 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock
 }

 static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo,
-                                 TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) {
+                                 TSDBKEY keyInBuf) {
  SDataBlockToLoadInfo info = {0};
-  getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader);
+  getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader);
  bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf ||
                            info.overlapWithDelInfo || info.overlapWithLastBlock);
  return isCleanFileBlock;
@ -1417,14 +1417,15 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
  return code;
 }

-static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo,
+static bool nextRowFromSttBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo,
                                 SVersionRange* pVerRange) {
  int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order) ? 1 : -1;

  while (1) {
    bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
    if (!hasVal) {  // the next value will be the accessed key in stt
-      pScanInfo->lastKeyInStt += step;
+      pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
+      pScanInfo->sttKeyInfo.nextProcKey += step;
      return false;
    }

@ -1433,10 +1434,11 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
    int64_t  ver = pRow->pBlockData->aVersion[pRow->iRow];

    pLastBlockReader->currentKey = key;
-    pScanInfo->lastKeyInStt = key;
+    pScanInfo->sttKeyInfo.nextProcKey = key;

    if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pLastBlockReader->order,
                        pVerRange)) {
+      pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;
      return true;
    }
  }
@ -1457,7 +1459,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas

  // avoid the fetch next row replace the referenced stt block in buffer
  doPinSttBlock(pLastBlockReader);
-  bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->info.verRange);
+  bool hasVal = nextRowFromSttBlocks(pLastBlockReader, pScanInfo, &pReader->info.verRange);
  doUnpinSttBlock(pLastBlockReader);
  if (hasVal) {
    int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
@ -1694,7 +1696,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
    }

    if (copied) {
-      pBlockScanInfo->lastKey = tsLastBlock;
+      pBlockScanInfo->lastProcKey = tsLastBlock;
      return TSDB_CODE_SUCCESS;
    } else {
      code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
@ -2062,9 +2064,9 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
  STbData* d = NULL;
  TSDBKEY  startKey = {0};
  if (ASCENDING_TRAVERSE(pReader->info.order)) {
-    startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey + 1, .version = pReader->info.verRange.minVer};
+    startKey = (TSDBKEY){.ts = pBlockScanInfo->lastProcKey + 1, .version = pReader->info.verRange.minVer};
  } else {
-    startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey - 1, .version = pReader->info.verRange.maxVer};
+    startKey = (TSDBKEY){.ts = pBlockScanInfo->lastProcKey - 1, .version = pReader->info.verRange.maxVer};
  }

  int32_t code =
@ -2129,9 +2131,9 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan

  STimeWindow w = pLBlockReader->window;
  if (ASCENDING_TRAVERSE(pLBlockReader->order)) {
-    w.skey = pScanInfo->lastKeyInStt;
+    w.skey = pScanInfo->sttKeyInfo.nextProcKey;
  } else {
-    w.ekey = pScanInfo->lastKeyInStt;
+    w.ekey = pScanInfo->sttKeyInfo.nextProcKey;
  }

  int64_t st = taosGetTimestampUs();
@ -2164,7 +2166,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
  initMemDataIterator(pScanInfo, pReader);
  initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost);

-  code = nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange);
+  code = nextRowFromSttBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange);

  int64_t el = taosGetTimestampUs() - st;
  pReader->cost.initLastBlockReader += (el / 1000.0);
@ -2209,7 +2211,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
  }

  if (copied) {
-    pBlockScanInfo->lastKey = key;
+    pBlockScanInfo->lastProcKey = key;
    return TSDB_CODE_SUCCESS;
  } else {
    TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
@ -2354,16 +2356,16 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
    TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);

    // it is a clean block, load it directly
-    if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf, pLastBlockReader) &&
-        (pRecord->numRow <= pReader->resBlockInfo.capacity)) {
-      if (asc || (!hasDataInLastBlock(pLastBlockReader))) {
+    int64_t cap = pReader->resBlockInfo.capacity;
+    if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) {
+      if (asc || (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) {
        code = copyBlockDataToSDataBlock(pReader);
        if (code) {
          goto _end;
        }

        // record the last key value
-        pBlockScanInfo->lastKey = asc ? pRecord->lastKey : pRecord->firstKey;
+        pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey;
        goto _end;
      }
    }
@ -2378,6 +2380,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
  }

  SBlockData* pBlockData = &pReader->status.fileBlockData;
+  initLastBlockReader(pLastBlockReader, pBlockScanInfo, pReader);

  while (1) {
    bool hasBlockData = false;
@ -2527,7 +2530,7 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader)
 static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) {
  SReaderStatus* pStatus = &pReader->status;
  pBlockNum->numOfBlocks = 0;
-  pBlockNum->numOfLastFiles = 0;
+  pBlockNum->numOfSttFiles = 0;

  size_t  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
  SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk));
@ -2564,7 +2567,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr
        return code;
      }

-      if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) {
+      if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) {
        break;
      }
    }
@ -2684,11 +2687,13 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
  }
 }

-static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, SLastBlockReader* pLastBlockReader, bool asc) {
-  if(!hasDataInLastBlock(pLastBlockReader)) {
+static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) {
+  ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT);
+
+  if(pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) {
    return true;
  } else {
-    int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+    int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
    return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt);
  }
 }
@ -2717,10 +2722,12 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
    return terrno;
  }

+  if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) {
    initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
-  TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader);
+  }

-  if (fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader)) {
+  TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader);
+  if (fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf)) {
    code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid);
    if (code != TSDB_CODE_SUCCESS) {
      return code;
@ -2728,13 +2735,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {

    // build composed data block
    code = buildComposedDataBlock(pReader);
-  } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pLastBlockReader, pReader->info.order)) {
+  } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pScanInfo, pReader->info.order)) {
    // data in memory that are earlier than current file block and stt blocks
    // rows in buffer should be less than the file block in asc, greater than file block in desc
-    int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pLastBlockReader, pReader->info.order);
+    int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pScanInfo, pReader->info.order);
    code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
  } else {
-    if (notOverlapWithSttFiles(pBlockInfo, pLastBlockReader, asc)) {
+    if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) {
      // whole block is required, return it directly
      SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
      pInfo->rows = pBlockInfo->record.numRow;
@ -2745,7 +2752,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
      setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);

      // update the last key for the corresponding table
-      pScanInfo->lastKey = asc ? pInfo->window.ekey : pInfo->window.skey;
+      pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey;
      tsdbDebug("%p uid:%" PRIu64
                " clean file block retrieved from file, global index:%d, "
                "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
@ -2760,8 +2767,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
      tsdbDebug("load data in last block firstly %s", pReader->idStr);
      int64_t st = taosGetTimestampUs();

+      // let's load data from stt files
+      initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
+
      // no data in last block, no need to proceed.
      while (hasDataInLastBlock(pLastBlockReader)) {
+        ASSERT(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA);
+
        code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
        if (code != TSDB_CODE_SUCCESS) {
          return code;
@ -2988,7 +3000,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter)
  if (pBlockInfo) {
    STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
    if (pScanInfo) {
-      lastKey = pScanInfo->lastKey;
+      lastKey = pScanInfo->lastProcKey;
    }

    pDumpInfo->totalRows = pBlockInfo->record.numRow;
@ -3013,7 +3025,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl
  }

  // all data files are consumed, try data in buffer
-  if (num.numOfBlocks + num.numOfLastFiles == 0) {
+  if (num.numOfBlocks + num.numOfSttFiles == 0) {
    pReader->status.loadFromFile = false;
    taosArrayDestroy(pTableList);
    return code;
@ -3140,9 +3152,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
  }
 }

-static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idStr,
+static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr,
                                  int8_t* pLevel) {
-  if (VND_IS_RSMA(pVnode)) {
+  if (VND_IS_RSMA(pVnode) && !pCond->skipRollup) {
    int8_t  level = 0;
    int8_t  precision = pVnode->config.tsdbCfg.precision;
    int64_t now = taosGetTimestamp(precision);
@ -3158,7 +3170,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
        }
        break;
      }
-      if ((now - pRetention->keep) <= (winSKey + offset)) {
+      if ((now - pRetention->keep) <= (pCond->twindows.skey + offset)) {
        break;
      }
      ++level;
@ -3458,15 +3470,15 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc

 int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
                               SRowMerger* pMerger, SVersionRange* pVerRange, const char* idStr) {
-  while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) {
+  while (nextRowFromSttBlocks(pLastBlockReader, pScanInfo, pVerRange)) {
    int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
    if (next1 == ts) {
      TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
      tsdbRowMergerAdd(pMerger, pRow1, NULL);
    } else {
      tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid,
-                pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt,
-                idStr);
+                pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline),
+                pScanInfo->sttKeyInfo.nextProcKey, idStr);
      break;
    }
  }
@ -3722,7 +3734,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT

  pBlock->info.dataLoad = 1;
  pBlock->info.rows += 1;
-  pScanInfo->lastKey = pTSRow->ts;
+  pScanInfo->lastProcKey = pTSRow->ts;
  return TSDB_CODE_SUCCESS;
 }

@ -3856,14 +3868,15 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t
    // todo extract method
    if (ASCENDING_TRAVERSE(pReader->info.order)) {
      int64_t skey = pReader->info.window.skey;
-      pInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey;
-      pInfo->lastKeyInStt = skey;
+      pInfo->lastProcKey = (skey > INT64_MIN) ? (skey - 1) : skey;
+      pInfo->sttKeyInfo.nextProcKey = skey;
    } else {
      int64_t ekey = pReader->info.window.ekey;
-      pInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
-      pInfo->lastKeyInStt = ekey;
+      pInfo->lastProcKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
+      pInfo->sttKeyInfo.nextProcKey = ekey;
    }

+    pInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
    tSimpleHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES);
  }

@ -4224,7 +4237,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
    if (pBlockScanInfo) {
      // save lastKey to restore memory iterator
      STimeWindow w = pReader->resBlockInfo.pResBlock->info.window;
-      pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey;
+      pBlockScanInfo->lastProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey;

      // reset current current table's data block scan info,
      pBlockScanInfo->iterInit = false;
--- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c
@ -157,17 +157,18 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf

    if (ASCENDING_TRAVERSE(pTsdbReader->info.order)) {
      int64_t skey = pTsdbReader->info.window.skey;
-      pScanInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey;
-      pScanInfo->lastKeyInStt = skey;
+      pScanInfo->lastProcKey = (skey > INT64_MIN) ? (skey - 1) : skey;
+      pScanInfo->sttKeyInfo.nextProcKey = skey;
    } else {
      int64_t ekey = pTsdbReader->info.window.ekey;
-      pScanInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
-      pScanInfo->lastKeyInStt = ekey;
+      pScanInfo->lastProcKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
+      pScanInfo->sttKeyInfo.nextProcKey = ekey;
    }

+    pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
    tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
    tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid,
-              pScanInfo->lastKey, pTsdbReader->idStr);
+              pScanInfo->lastProcKey, pTsdbReader->idStr);
  }

  taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);
@ -200,8 +201,8 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
    }

    pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline);
-    pInfo->lastKey = ts;
-    pInfo->lastKeyInStt = ts + step;
+    pInfo->lastProcKey = ts;
+    pInfo->sttKeyInfo.nextProcKey = ts + step;
  }
 }

@ -241,6 +242,7 @@ static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) {
  taosArrayClear(pScanInfo->pBlockList);
  taosArrayClear(pScanInfo->pBlockIdxList);
  taosArrayClear(pScanInfo->pFileDelData);  // del data from each file set
+  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
 }

 void cleanupInfoFoxNextFileset(SSHashObj* pTableMap) {
--- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h
+++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h
@ -63,10 +63,21 @@ typedef struct STableDataBlockIdx {
  int32_t globalIndex;
 } STableDataBlockIdx;

+typedef enum ESttKeyStatus {
+  STT_FILE_READER_UNINIT = 0x0,
+  STT_FILE_NO_DATA = 0x1,
+  STT_FILE_HAS_DATA = 0x2,
+} ESttKeyStatus;
+
+typedef struct SSttKeyInfo {
+  ESttKeyStatus status;           // this value should be updated when switch to the next fileset
+  int64_t       nextProcKey;
+} SSttKeyInfo;
+
 typedef struct STableBlockScanInfo {
  uint64_t    uid;
-  TSKEY     lastKey;
-  TSKEY     lastKeyInStt;       // last accessed key in stt
+  TSKEY       lastProcKey;
+  SSttKeyInfo sttKeyInfo;
  SArray*     pBlockList;        // block data index list, SArray<SBrinRecord>
  SArray*     pBlockIdxList;     // SArray<STableDataBlockIndx>
  SArray*     pMemDelData;       // SArray<SDelData>
@ -108,7 +119,7 @@ typedef struct STableUidList {

 typedef struct {
  int32_t numOfBlocks;
-  int32_t numOfLastFiles;
+  int32_t numOfSttFiles;
 } SBlockNumber;

 typedef struct SBlockIndex {
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@ -13,8 +13,8 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "cos.h"
 #include "tsdb.h"
-#include "vndCos.h"

 static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
  int32_t     code = 0;
@ -338,9 +338,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
    // 2, retrieve pgs from s3
    uint8_t *pBlock = NULL;
    int64_t  retrieve_offset = PAGE_OFFSET(pgno, pFD->szPage);
-    int64_t  pgnoEnd = pgno - 1 + (size - n + szPgCont - 1) / szPgCont;
+    int64_t  pgnoEnd = pgno - 1 + (bOffset + size - n + szPgCont - 1) / szPgCont;
    int64_t  retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
-    code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, &pBlock);
+    code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
    if (code != TSDB_CODE_SUCCESS) {
      goto _exit;
    }
--- a/source/dnode/vnode/src/tsdb/tsdbRetention.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@ -13,9 +13,10 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "cos.h"
 #include "tsdb.h"
 #include "tsdbFS2.h"
-#include "vndCos.h"
+#include "vnd.h"

 typedef struct {
  STsdb  *tsdb;
@ -291,15 +292,15 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
  if (expLevel < 0) {  // remove the fileset
    for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) {
      if (fobj == NULL) continue;
-
+      /*
      int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
      if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
        code = tsdbRemoveFileObjectS3(rtner, fobj);
        TSDB_CHECK_CODE(code, lino, _exit);
-      } else {
+        } else {*/
      code = tsdbDoRemoveFileObject(rtner, fobj);
      TSDB_CHECK_CODE(code, lino, _exit);
-      }
+      //}
    }

    SSttLvl *lvl;
@ -387,6 +388,8 @@ _exit:
  return code;
 }

+static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
+
 static int32_t tsdbDoRetentionSync(void *arg) {
  int32_t code = 0;
  int32_t lino = 0;
@ -409,6 +412,7 @@ _exit:
    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
  }
  tsem_post(&((SRtnArg *)arg)->tsdb->pVnode->canCommit);
+  tsdbFreeRtnArg(arg);
  return code;
 }

@ -438,7 +442,7 @@ _exit:
  return code;
 }

-static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
+

 int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
  int32_t code = 0;
--- a/source/dnode/vnode/src/tsdb/tsdbWrite.c
+++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c
@ -13,8 +13,8 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "cos.h"
 #include "tsdb.h"
-#include "vndCos.h"

 /**
 * @brief max key by precision
@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2
  arrSize = taosArrayGetSize(pMsg->aSubmitTbData);

  // scan and convert
-  if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
+  if ((terrno = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) {
    if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
      tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno));
    }
--- a/source/dnode/vnode/src/vnd/vnodeModule.c
+++ b/source/dnode/vnode/src/vnd/vnodeModule.c
@ -13,8 +13,8 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "cos.h"
 #include "vnd.h"
-#include "vndCos.h"

 typedef struct SVnodeTask SVnodeTask;
 struct SVnodeTask {
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@ -13,10 +13,10 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "cos.h"
 #include "sync.h"
 #include "tsdb.h"
 #include "vnd.h"
-#include "vndCos.h"

 int32_t vnodeGetPrimaryDir(const char *relPath, int32_t diskPrimary, STfs *pTfs, char *buf, size_t bufLen) {
  if (pTfs) {
--- a/source/dnode/vnode/src/vnd/vnodeRetention.c
+++ b/source/dnode/vnode/src/vnd/vnodeRetention.c
@ -15,8 +15,12 @@

 #include "vnd.h"

-extern int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync);
-
 int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
-  return tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
+
+  if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
+
+  return code;
 }
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@ -18,7 +18,7 @@
 #include "tmsg.h"
 #include "tstrbuild.h"
 #include "vnd.h"
-#include "vndCos.h"
+#include "cos.h"
 #include "vnode.h"
 #include "vnodeInt.h"

@ -928,6 +928,17 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
      goto _exit;
    }

+    if(tsEnableAuditCreateTable){
+      char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
+      if (str == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        rcode = -1;
+        goto _exit;
+      }
+      strcpy(str, pCreateReq->name);
+      taosArrayPush(tbNames, &str);
+    }
+
    // validate hash
    sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name);
    if (vnodeValidateTableHash(pVnode, tbName) < 0) {
@ -951,12 +962,6 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
    }

    taosArrayPush(rsp.pArray, &cRsp);
-
-    if (tsEnableAuditCreateTable) {
-      char *str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
-      strcpy(str, pCreateReq->name);
-      taosArrayPush(tbNames, &str);
-    }
  }

  vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids));
@ -985,10 +990,10 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
    tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);

    SStringBuilder sb = {0};
-    for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
-      char **key = (char **)taosArrayGet(tbNames, iReq);
+    for(int32_t i = 0; i < tbNames->size; i++){
+      char** key = (char**)taosArrayGet(tbNames, i);
      taosStringBuilderAppendStringLen(&sb, *key, strlen(*key));
-      if (iReq < req.nReqs - 1) {
+      if(i < tbNames->size - 1){
        taosStringBuilderAppendChar(&sb, ',');
      }
      taosMemoryFreeClear(*key);
@ -1669,7 +1674,7 @@ _exit:
  atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
  if (code == 0) {
    atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
-    tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
+    code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
  }

  // clear
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@ -557,7 +557,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
  SStreamMeta* pMeta = pVnode->pTq->pStreamMeta;
  streamMetaWLock(pMeta);

-  if (pMeta->startInfo.startAllTasksFlag) {
+  if (pMeta->startInfo.tasksWillRestart) {
    vInfo("vgId:%d, sync restore finished, stream tasks will be launched by other thread", vgId);
    streamMetaWUnLock(pMeta);
    return;
@ -570,7 +570,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
    } else {
      vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId);
      tqResetStreamTaskStatus(pVnode->pTq);
-      tqLaunchStreamTaskAsync(pVnode->pTq);
+      tqStartStreamTaskAsync(pVnode->pTq, false);
    }
  } else {
    vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId);
--- a/source/libs/command/inc/commandInt.h
+++ b/source/libs/command/inc/commandInt.h
@ -112,7 +112,6 @@ extern "C" {
 #define COMMAND_CATALOG_DEBUG "catalogDebug"
 #define COMMAND_ENABLE_MEM_DEBUG "enableMemDebug"
 #define COMMAND_DISABLE_MEM_DEBUG "disableMemDebug"
-#define COMMAND_ASYNCLOG          "asynclog"

 typedef struct SExplainGroup {
  int32_t   nodeNum;
--- a/source/libs/command/src/command.c
+++ b/source/libs/command/src/command.c
@ -808,16 +808,6 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) {
      return code;
    }
    qInfo("memory dbg disabled");
-  } else if (0 == strcasecmp(cmd, COMMAND_ASYNCLOG)) {
-    int newAsyncLogValue = (strlen(value) == 0) ? 1 : atoi(value);
-    if (newAsyncLogValue != 0 && newAsyncLogValue != 1) {
-      code = TSDB_CODE_INVALID_CFG_VALUE;
-      qError("failed to alter asynclog, error:%s", tstrerror(code));
-      goto _return;
-    }
-
-    code = TSDB_CODE_SUCCESS;
-    tsAsyncLog = newAsyncLogValue;
  } else {
    goto _return;
  }
@ -844,10 +834,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) {
    goto _return;
  }

-  bool forbidden = false;
-  taosLocalCfgForbiddenToChange(pStmt->config, &forbidden);
-  if (forbidden) {
-    terrno = TSDB_CODE_OPS_NOT_SUPPORT;
+  if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false)) {
    return terrno;
  }

--- a/source/libs/executor/inc/executil.h
+++ b/source/libs/executor/inc/executil.h
@ -40,7 +40,9 @@
 #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))

 typedef struct SGroupResInfo {
-  int32_t index;
+  int32_t index;    // rows consumed in func:doCopyToSDataBlockXX
+  int32_t iter;     // relate to index-1, last consumed data's slot id in hash table
+  void*   dataPos;  // relate to index-1, last consumed data's position, in the nodelist of cur slot
  SArray* pRows;    // SArray<SResKeyPos>
  char*   pBuf;
  bool    freeItem;
@ -178,7 +180,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
 SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
 SColumn   extractColumnFromColumnNode(SColumnNode* pColNode);

-int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
+int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode, const SReadHandle* readHandle);
 void    cleanupQueryTableDataCond(SQueryTableDataCond* pCond);

 int32_t convertFillType(int32_t mode);
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@ -580,6 +580,7 @@ typedef struct SStreamSessionAggOperatorInfo {
  bool                reCkBlock;
  SSDataBlock*        pCheckpointRes;
  bool                clearState;
+  bool                recvGetAll;
 } SStreamSessionAggOperatorInfo;

 typedef struct SStreamStateAggOperatorInfo {
@ -603,6 +604,7 @@ typedef struct SStreamStateAggOperatorInfo {
  SArray*             historyWins;
  bool                reCkBlock;
  SSDataBlock*        pCheckpointRes;
+  bool                recvGetAll;
 } SStreamStateAggOperatorInfo;

 typedef struct SStreamPartitionOperatorInfo {
@ -677,6 +679,12 @@ void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
 void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
                            SDiskbasedBuf* pBuf);

+/**
+ * @brief copydata from hash table, instead of copying from SGroupResInfo's pRow
+ */
+int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
+                           SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup);
+
 bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
 bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo);
 void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@ -54,6 +54,19 @@ static int32_t      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM

 #define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)

+static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) {
+  SNode*  pNode;
+  int32_t idx = 0;
+  FOREACH(pNode, pTargets) {
+    if (nodeType(pNode) == QUERY_NODE_COLUMN) {
+      SColumnNode* pCol = (SColumnNode*)pNode;
+      SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx);
+      pColInfo->info.colId = pCol->colId;
+    }
+    idx++;
+  }
+}
+
 SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
                                           STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
  int32_t             code = TSDB_CODE_SUCCESS;
@ -114,10 +127,12 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
    capacity = TMIN(totalTables, 4096);

    pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false);
+    setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode->pTargets);
    blockDataEnsureCapacity(pInfo->pBufferredRes, capacity);
  } else {  // by tags
    pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull);
    capacity = 1;  // only one row output
+    setColIdForCacheReadBlock(pInfo->pRes, pScanNode->pTargets);
  }

  initResultSizeInfo(&pOperator->resultInfo, capacity);
@ -191,9 +206,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
    SSDataBlock* pRes = pInfo->pRes;

    if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) {
-      for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
-        SColMatchItem* pMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i);
-        int32_t        slotId = pMatchInfo->dstSlotId;
+      for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBufferredRes->pDataBlock); ++i) {
+        SColumnInfoData* pCol = taosArrayGet(pRes->pDataBlock, i);
+        int32_t        slotId = pCol->info.slotId;

        SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId);
        SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId);
@ -201,10 +216,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
        if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) {
          colDataSetNULL(pDst, 0);
        } else {
+          if (pSrc->pData) {
            char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes);
            colDataSetVal(pDst, 0, p, false);
          }
        }
+      }

      pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes);
      pRes->info.rows = 1;
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@ -1713,7 +1713,7 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
  return c;
 }

-int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
+int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode, const SReadHandle* readHandle) {
  pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
  pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);

@ -1732,6 +1732,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
  pCond->type = TIMEWINDOW_RANGE_CONTAINED;
  pCond->startVersion = -1;
  pCond->endVersion = -1;
+  pCond->skipRollup = readHandle->skipRollup;

  int32_t j = 0;
  for (int32_t i = 0; i < pCond->numOfCols; ++i) {
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@ -69,12 +69,14 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf
    } else if (type == STREAM_INPUT__DATA_BLOCK) {
      for (int32_t i = 0; i < numOfBlocks; ++i) {
        SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
-        SPackedData  tmp = {
-             .pDataBlock = pDataBlock,
-        };
+        SPackedData tmp = {.pDataBlock = pDataBlock};
        taosArrayPush(pInfo->pBlockLists, &tmp);
      }
      pInfo->blockType = STREAM_INPUT__DATA_BLOCK;
+    } else if (type == STREAM_INPUT__CHECKPOINT) {
+      SPackedData tmp = {.pDataBlock = input};
+      taosArrayPush(pInfo->pBlockLists, &tmp);
+      pInfo->blockType = STREAM_INPUT__CHECKPOINT;
    }

    return TSDB_CODE_SUCCESS;
@ -633,7 +635,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
    blockIndex += 1;

    current += p->info.rows;
-    ASSERT(p->info.rows > 0);
+    ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT);
    taosArrayPush(pResList, &p);

    if (current >= rowsThreshold) {
@ -871,32 +873,6 @@ int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList) {
  return getOperatorExplainExecInfo(pTaskInfo->pRoot, pExecInfoList);
 }

-int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
-  SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo;
-  if (pTaskInfo->pRoot == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
-
-  int32_t nOptrWithVal = 0;
-  //  int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
-  //  if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) {
-  //    taosMemoryFreeClear(*pOutput);
-  //    *len = 0;
-  //  }
-  return 0;
-}
-
-int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {
-  SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo;
-
-  if (pTaskInfo == NULL || pInput == NULL || len == 0) {
-    return TSDB_CODE_INVALID_PARA;
-  }
-
-  return 0;
-  //  return decodeOperator(pTaskInfo->pRoot, pInput, len);
-}
-
 int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
  SOperatorInfo* pOperator = pTaskInfo->pRoot;
@ -1072,7 +1048,7 @@ int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) {
  }
 }

-bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) {
+bool qStreamScanhistoryFinished(qTaskInfo_t tinfo) {
  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
  return pTaskInfo->streamInfo.recoverScanFinished;
 }
--- a/source/libs/executor/src/executorInt.c
+++ b/source/libs/executor/src/executorInt.c
@ -655,6 +655,85 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
  return 0;
 }

+int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
+                                 SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold,
+                                 bool ignoreGroup) {
+  SExprInfo*      pExprInfo = pSup->pExprInfo;
+  int32_t         numOfExprs = pSup->numOfExprs;
+  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;
+  SqlFunctionCtx* pCtx = pSup->pCtx;
+
+  size_t  keyLen = 0;
+  int32_t numOfRows = tSimpleHashGetSize(pHashmap);
+
+  // begin from last iter
+  void*   pData = pGroupResInfo->dataPos;
+  int32_t iter = pGroupResInfo->iter;
+  while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) {
+    void*               key = tSimpleHashGetKey(pData, &keyLen);
+    SResultRowPosition* pos = pData;
+    uint64_t            groupId = *(uint64_t*)key;
+
+    SFilePage* page = getBufPage(pBuf, pos->pageId);
+    if (page == NULL) {
+      qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo));
+      T_LONG_JMP(pTaskInfo->env, terrno);
+    }
+
+    SResultRow* pRow = (SResultRow*)((char*)page + pos->offset);
+
+    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
+
+    // no results, continue to check the next one
+    if (pRow->numOfRows == 0) {
+      pGroupResInfo->index += 1;
+      pGroupResInfo->iter = iter;
+      pGroupResInfo->dataPos = pData;
+
+      releaseBufPage(pBuf, page);
+      continue;
+    }
+
+    if (!ignoreGroup) {
+      if (pBlock->info.id.groupId == 0) {
+        pBlock->info.id.groupId = groupId;
+      } else {
+        // current value belongs to different group, it can't be packed into one datablock
+        if (pBlock->info.id.groupId != groupId) {
+          releaseBufPage(pBuf, page);
+          break;
+        }
+      }
+    }
+
+    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
+      uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - iter) > 1 ? 1 : 0);
+      blockDataEnsureCapacity(pBlock, newSize);
+      qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize,
+             pBlock->info.capacity, GET_TASKID(pTaskInfo));
+      // todo set the pOperator->resultInfo size
+    }
+
+    pGroupResInfo->index += 1;
+    pGroupResInfo->iter = iter;
+    pGroupResInfo->dataPos = pData;
+
+    copyResultrowToDataBlock(pExprInfo, numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo);
+
+    releaseBufPage(pBuf, page);
+    pBlock->info.rows += pRow->numOfRows;
+    if (pBlock->info.rows >= threshold) {
+      break;
+    }
+  }
+
+  qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
+        pBlock->info.id.groupId);
+  pBlock->info.dataLoad = 1;
+  blockDataUpdateTsWindow(pBlock, 0);
+  return 0;
+}
+
 int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
                           SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup) {
  SExprInfo*      pExprInfo = pSup->pExprInfo;
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@ -370,6 +370,72 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
  return (pRes->info.rows == 0) ? NULL : pRes;
 }

+bool hasRemainResultByHash(SOperatorInfo* pOperator) {
+  SGroupbyOperatorInfo* pInfo = pOperator->info;
+  SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable;
+  return pInfo->groupResInfo.index < tSimpleHashGetSize(pHashmap);
+}
+
+void doBuildResultDatablockByHash(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
+                                  SDiskbasedBuf* pBuf) {
+  SGroupbyOperatorInfo* pInfo = pOperator->info;
+  SSHashObj*            pHashmap = pInfo->aggSup.pResultRowHashTable;
+  SExecTaskInfo*        pTaskInfo = pOperator->pTaskInfo;
+
+  SSDataBlock* pBlock = pInfo->binfo.pRes;
+
+  // set output datablock version
+  pBlock->info.version = pTaskInfo->version;
+
+  blockDataCleanup(pBlock);
+  if (!hasRemainResultByHash(pOperator)) {
+    return;
+  }
+
+  pBlock->info.id.groupId = 0;
+  if (!pInfo->binfo.mergeResultBlock) {
+    doCopyToSDataBlockByHash(pTaskInfo, pBlock, &pOperator->exprSupp, pInfo->aggSup.pResultBuf, &pInfo->groupResInfo,
+                             pHashmap, pOperator->resultInfo.threshold, false);
+  } else {
+    while (hasRemainResultByHash(pOperator)) {
+      doCopyToSDataBlockByHash(pTaskInfo, pBlock, &pOperator->exprSupp, pInfo->aggSup.pResultBuf, &pInfo->groupResInfo,
+                               pHashmap, pOperator->resultInfo.threshold, true);
+      if (pBlock->info.rows >= pOperator->resultInfo.threshold) {
+        break;
+      }
+      pBlock->info.id.groupId = 0;
+    }
+
+    // clear the group id info in SSDataBlock, since the client does not need it
+    pBlock->info.id.groupId = 0;
+  }
+}
+
+static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) {
+  SGroupbyOperatorInfo* pInfo = pOperator->info;
+  SSDataBlock* pRes = pInfo->binfo.pRes;
+
+  // after filter, if result block turn to null, get next from whole set
+  while (1) {
+    doBuildResultDatablockByHash(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+
+    doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
+    if (!hasRemainResultByHash(pOperator)) {
+      setOperatorCompleted(pOperator);
+      // clean hash after completed
+      tSimpleHashCleanup(pInfo->aggSup.pResultRowHashTable);
+      pInfo->aggSup.pResultRowHashTable = NULL;
+      break;
+    }
+    if (pRes->info.rows > 0) {
+      break;
+    }
+  }
+
+  pOperator->resultInfo.totalRows += pRes->info.rows;
+  return (pRes->info.rows == 0) ? NULL : pRes;
+}
+
 static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
  if (pOperator->status == OP_EXEC_DONE) {
    return NULL;
@ -379,8 +445,9 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {

  SGroupbyOperatorInfo* pInfo = pOperator->info;
  if (pOperator->status == OP_RES_TO_RETURN) {
-    return buildGroupResultDataBlock(pOperator);
+    return buildGroupResultDataBlockByHash(pOperator);
  }
+  SGroupResInfo* pGroupResInfo = &pInfo->groupResInfo;
  
  int32_t order = pInfo->binfo.inputTsOrder;
  int64_t        st = taosGetTimestampUs();
@ -425,10 +492,20 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
    }
  }
 #endif
-  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0);
+  // initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0);
+  if (pGroupResInfo->pRows != NULL) {
+    taosArrayDestroy(pGroupResInfo->pRows);
+  }
+  if (pGroupResInfo->pBuf) {
+    taosMemoryFree(pGroupResInfo->pBuf);
+    pGroupResInfo->pBuf = NULL;
+  }
+  pGroupResInfo->index = 0;
+  pGroupResInfo->iter = 0;
+  pGroupResInfo->dataPos = NULL;

  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
-  return buildGroupResultDataBlock(pOperator);
+  return buildGroupResultDataBlockByHash(pOperator);
 }

 SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo) {
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@ -1035,7 +1035,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
  }

  initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->base.limitInfo);
-  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode);
+  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
  }
@ -3533,7 +3533,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
    goto _error;
  }

-  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode);
+  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle);
  if (code != TSDB_CODE_SUCCESS) {
    taosArrayDestroy(pInfo->base.matchInfo.pList);
    goto _error;
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@ -373,8 +373,11 @@ void clearGroupResInfo(SGroupResInfo* pGroupResInfo) {
  if (pGroupResInfo->freeItem) {
    int32_t size = taosArrayGetSize(pGroupResInfo->pRows);
    for (int32_t i = pGroupResInfo->index; i < size; i++) {
-      void* pVal = taosArrayGetP(pGroupResInfo->pRows, i);
-      taosMemoryFree(pVal);
+      SRowBuffPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
+      if (!pPos->needFree && !pPos->pRowBuff) {
+        taosMemoryFreeClear(pPos->pKey);
+        taosMemoryFree(pPos);
+      }
    }
    pGroupResInfo->freeItem = false;
  }
@ -2532,6 +2535,15 @@ void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
  taosMemoryFree(buf);
 }

+static void resetUnCloseSessionWinInfo(SSHashObj* winMap) {
+  void*   pIte = NULL;
+  int32_t iter = 0;
+  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {
+    SResultWindowInfo* pResInfo = pIte;
+    pResInfo->pStatePos->beUsed = true;
+  }
+}
+
 static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
  SExprSupp*                     pSup = &pOperator->exprSupp;
  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
@ -2546,6 +2558,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
    if (opRes) {
      return opRes;
    }
+
+    if (pInfo->recvGetAll) {
+      pInfo->recvGetAll = false;
+      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);
+    }
+
    setOperatorCompleted(pOperator);
    return NULL;
  }
@ -2583,6 +2601,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
      taosArrayDestroy(pWins);
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
+      pInfo->recvGetAll = true;
      getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);
      continue;
    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
@ -2838,6 +2857,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh

  pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
  pInfo->clearState = false;
+  pInfo->recvGetAll = false;
+
  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
    // for stream
  void*   buff = NULL;
@ -3454,6 +3475,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
      return resBlock;
    }

+    if (pInfo->recvGetAll) {
+      pInfo->recvGetAll = false;
+      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);
+    }
+
    setOperatorCompleted(pOperator);
    return NULL;
  }
@ -3482,6 +3508,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
      taosArrayDestroy(pWins);
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
+      pInfo->recvGetAll = true;
      getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);
      continue;
    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
@ -3712,6 +3739,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
  }

  pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
+  pInfo->recvGetAll = false;

  // for stream
  void*   buff = NULL;
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@ -1989,7 +1989,7 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
  }
  if (para2Bytes <= 0 || para2Bytes > 4096) {  // cast dst var type length limits to 4096 bytes
    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "CAST function converted length should be in range [0, 4096] bytes");
+                           "CAST function converted length should be in range (0, 4096] bytes");
  }

  // add database precision as param
@ -2772,7 +2772,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
  {
    .name = "_cache_last",
    .type = FUNCTION_TYPE_CACHE_LAST,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
    .translateFunc = translateFirstLast,
    .getEnvFunc   = getFirstLastFuncEnv,
    .initFunc     = functionSetup,
@ -3312,26 +3312,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
    .sprocessFunc = castFunction,
    .finalizeFunc = NULL
  },
-  {
-    .name = "to_timestamp",
-    .type = FUNCTION_TYPE_TO_TIMESTAMP,
-    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC,
-    .translateFunc = translateToTimestamp,
-    .getEnvFunc = NULL,
-    .initFunc = NULL,
-    .sprocessFunc = toTimestampFunction,
-    .finalizeFunc = NULL
-  },
-  {
-    .name = "to_char",
-    .type = FUNCTION_TYPE_TO_CHAR,
-    .classification = FUNC_MGT_SCALAR_FUNC,
-    .translateFunc = translateToChar,
-    .getEnvFunc = NULL,
-    .initFunc = NULL,
-    .sprocessFunc = toCharFunction,
-    .finalizeFunc = NULL
-  },
  {
    .name = "to_iso8601",
    .type = FUNCTION_TYPE_TO_ISO8601,
@ -3709,6 +3689,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
    .sprocessFunc = qVgIdFunction,
    .finalizeFunc = NULL
  },
+  {
+    .name = "to_timestamp",
+    .type = FUNCTION_TYPE_TO_TIMESTAMP,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC,
+    .translateFunc = translateToTimestamp,
+    .getEnvFunc = NULL,
+    .initFunc = NULL,
+    .sprocessFunc = toTimestampFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "to_char",
+    .type = FUNCTION_TYPE_TO_CHAR,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateToChar,
+    .getEnvFunc = NULL,
+    .initFunc = NULL,
+    .sprocessFunc = toCharFunction,
+    .finalizeFunc = NULL
+  },
  
 };
 // clang-format on
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@ -1773,6 +1773,7 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {

 static const char* jkLastRowScanPhysiPlanGroupTags = "GroupTags";
 static const char* jkLastRowScanPhysiPlanGroupSort = "GroupSort";
+static const char* jkLastRowScanPhysiPlanTargets = "Targets";

 static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
  const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj;
@ -1784,6 +1785,9 @@ static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
  if (TSDB_CODE_SUCCESS == code) {
    code = tjsonAddBoolToObject(pJson, jkLastRowScanPhysiPlanGroupSort, pNode->groupSort);
  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets);
+  }

  return code;
 }
@ -1798,6 +1802,9 @@ static int32_t jsonToPhysiLastRowScanNode(const SJson* pJson, void* pObj) {
  if (TSDB_CODE_SUCCESS == code) {
    code = tjsonGetBoolValue(pJson, jkLastRowScanPhysiPlanGroupSort, &pNode->groupSort);
  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkLastRowScanPhysiPlanTargets, &pNode->pTargets);
+  }

  return code;
 }
--- a/source/libs/nodes/src/nodesMsgFuncs.c
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@ -2052,7 +2052,8 @@ enum {
  PHY_LAST_ROW_SCAN_CODE_SCAN = 1,
  PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS,
  PHY_LAST_ROW_SCAN_CODE_GROUP_SORT,
-  PHY_LAST_ROW_SCAN_CODE_IGNULL
+  PHY_LAST_ROW_SCAN_CODE_IGNULL,
+  PHY_LAST_ROW_SCAN_CODE_TARGETS
 };

 static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2068,6 +2069,9 @@ static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder
  if (TSDB_CODE_SUCCESS == code) {
    code = tlvEncodeBool(pEncoder, PHY_LAST_ROW_SCAN_CODE_IGNULL, pNode->ignoreNull);
  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tlvEncodeObj(pEncoder, PHY_LAST_ROW_SCAN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
+  }

  return code;
 }
@ -2091,6 +2095,9 @@ static int32_t msgToPhysiLastRowScanNode(STlvDecoder* pDecoder, void* pObj) {
      case PHY_LAST_ROW_SCAN_CODE_IGNULL:
        code = tlvDecodeBool(pTlv, &pNode->ignoreNull);
        break;
+      case PHY_LAST_ROW_SCAN_CODE_TARGETS:
+        code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
+        break;
      default:
        break;
    }
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@ -1285,6 +1285,7 @@ void nodesDestroyNode(SNode* pNode) {
      SLastRowScanPhysiNode* pPhyNode = (SLastRowScanPhysiNode*)pNode;
      destroyScanPhysiNode((SScanPhysiNode*)pNode);
      nodesDestroyList(pPhyNode->pGroupTags);
+      nodesDestroyList(pPhyNode->pTargets);
      break;
    }
    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
--- a/source/libs/parser/src/parCalcConst.c
+++ b/source/libs/parser/src/parCalcConst.c
@ -176,12 +176,15 @@ static int32_t calcConstStmtCondition(SCalcConstContext* pCxt, SNode** pCond, bo
 static EDealRes doFindAndReplaceNode(SNode** pNode, void* pContext) {
  SCalcConstContext* pCxt = pContext;
  if (pCxt->replaceCxt.pTarget == *pNode) {
+    char aliasName[TSDB_COL_NAME_LEN] = {0};
+    strcpy(aliasName, ((SExprNode*)*pNode)->aliasName);
    nodesDestroyNode(*pNode);
    *pNode = nodesCloneNode(pCxt->replaceCxt.pNew);
    if (NULL == *pNode) {
      pCxt->code = TSDB_CODE_OUT_OF_MEMORY;
      return DEAL_RES_ERROR;
    }
+    strcpy(((SExprNode*)*pNode)->aliasName, aliasName);

    pCxt->replaceCxt.replaced = true;
    return DEAL_RES_END;
@ -211,7 +214,6 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d
  }

  char aliasName[TSDB_COL_NAME_LEN] = {0};
-  strcpy(aliasName, ((SExprNode*)pProject)->aliasName);
  int32_t code = TSDB_CODE_SUCCESS;
  if (dual) {
    code = scalarCalculateConstantsFromDual(pProject, pNew);
@ -219,15 +221,20 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d
    code = scalarCalculateConstants(pProject, pNew);
  }
  if (TSDB_CODE_SUCCESS == code) {
-    strcpy(((SExprNode*)*pNew)->aliasName, aliasName);
    if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) {
      int32_t size = taosArrayGetSize(pAssociation);
      for (int32_t i = 0; i < size; ++i) {
        SAssociationNode* pAssNode = taosArrayGet(pAssociation, i);
        SNode** pCol = pAssNode->pPlace;
        if (*pCol == pAssNode->pAssociationNode) {
+          strcpy(aliasName, ((SExprNode*)*pCol)->aliasName);
+          SArray* pOrigAss = NULL;
+          TSWAP(((SExprNode*)*pCol)->pAssociation, pOrigAss);
          nodesDestroyNode(*pCol);
          *pCol = nodesCloneNode(*pNew);
+          TSWAP(pOrigAss, ((SExprNode*)*pCol)->pAssociation);
+          taosArrayDestroy(pOrigAss);
+          strcpy(((SExprNode*)*pCol)->aliasName, aliasName);
          if (NULL == *pCol) {
            code = TSDB_CODE_OUT_OF_MEMORY;
            break;
@ -285,7 +292,7 @@ static SNode* createConstantValue() {
 static int32_t calcConstProjections(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) {
  SNode* pProj = NULL;
  WHERE_EACH(pProj, pSelect->pProjectionList) {
-    if (subquery && !pSelect->isDistinct && isUselessCol((SExprNode*)pProj)) {
+    if (subquery && !pSelect->isDistinct && !pSelect->tagScan && isUselessCol((SExprNode*)pProj)) {
      ERASE_NODE(pSelect->pProjectionList);
      continue;
    }
--- a/source/libs/parser/src/parInsertSql.c
+++ b/source/libs/parser/src/parInsertSql.c
@ -1584,8 +1584,7 @@ typedef union SRowsDataContext{
  SStbRowsDataContext* pStbRowsCxt;
 } SRowsDataContext;

-static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, 
-                                char* ctbName, bool* pFoundCtbName) {
+static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) {
  *pFoundCtbName = false;
  int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
  if (code == TSDB_CODE_SUCCESS){
@ -1595,7 +1594,13 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext*

    if (pToken->n > 0) {
      if (pToken->n <= TSDB_TABLE_NAME_LEN - 1) {
-        memcpy(pStbRowsCxt->ctbName.tname, pToken->z, pToken->n);
+        for (int i = 0; i < pToken->n; ++i) {
+          if (pToken->z[i] == '.') {
+            return buildInvalidOperationMsg(&pCxt->msg, "tbname can not contain '.'");
+          } else {
+            pStbRowsCxt->ctbName.tname[i] = pToken->z[i]; 
+          }
+        }
        pStbRowsCxt->ctbName.tname[pToken->n] = '\0';
        *pFoundCtbName = true;
      } else {
@ -1677,8 +1682,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
      }
    }
    else if (pCols->pColIndex[i] == tbnameIdx) {
-      char ctbName[TSDB_TABLE_NAME_LEN];
-      code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, ctbName, bFoundTbName);
+      code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, bFoundTbName);
    }

    if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) {
@ -1757,7 +1761,7 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod
                             .requestId = pCxt->pComCxt->requestId,
                             .requestObjRefId = pCxt->pComCxt->requestRid,
                             .mgmtEps = pCxt->pComCxt->mgmtEpSet};
-    code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg);
+    code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStbRowsCxt->ctbName, &vg);
    if (code == TSDB_CODE_SUCCESS) {
      taosHashPut(pStmt->pVgroupsHashObj, (const char*)(&vg.vgId), sizeof(vg.vgId), &vg, sizeof(vg));
      pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1;
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@ -266,7 +266,7 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols

  if (NULL == pScanCols) {
    if (NULL == pScanPseudoCols) {
-      return SCAN_TYPE_TABLE;
+      return (!tagScan) ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG;
    }
    return FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType
               ? SCAN_TYPE_BLOCK_INFO
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@ -2478,6 +2478,27 @@ static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast
  return false;
 }

+/// @brief check if we can apply last row scan optimization
+/// @param lastColNum how many distinct last col specified
+/// @param lastColId only used when lastColNum equals 1, the col id of the only one last col
+/// @param selectNonPKColNum num of normal cols
+/// @param selectNonPKColId only used when selectNonPKColNum equals 1, the col id of the only one select col
+static bool lastRowScanOptCheckColNum(int32_t lastColNum, col_id_t lastColId,
+                                      int32_t selectNonPKColNum, col_id_t selectNonPKColId) {
+  // multi select non pk col + last func: select c1, c2, last(c1)
+  if (selectNonPKColNum > 1 && lastColNum > 0) return false;
+
+  if (selectNonPKColNum == 1) {
+    // select last(c1), last(c2), c1 ...
+    // which is not possible currently
+    if (lastColNum > 1) return false;
+
+    // select last(c1), c2 ...
+    if (lastColNum == 1 && lastColId != selectNonPKColId) return false;
+  }
+  return true;
+}
+
 static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
  if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) ||
      QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) {
@ -2493,9 +2514,10 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
    return false;
  }

-  bool   hasLastFunc = false;
-  bool   hasSelectFunc = false;
+  bool     hasNonPKSelectFunc = false;
  SNode*   pFunc = NULL;
+  int32_t  lastColNum = 0, selectNonPKColNum = 0;
+  col_id_t lastColId = -1, selectNonPKColId = -1;
  FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) {
    SFunctionNode* pAggFunc = (SFunctionNode*)pFunc;
    if (FUNCTION_TYPE_LAST == pAggFunc->funcType) {
@ -2505,16 +2527,33 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
        if (pCol->colType != COLUMN_TYPE_COLUMN) {
          return false;
        }
+        if (lastColId != pCol->colId) {
+          lastColId = pCol->colId;
+          lastColNum++;
        }
-      if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) {
+      }
+      if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) {
        return false;
      }
-      hasLastFunc = true;
+      if (!lastRowScanOptCheckColNum(lastColNum, lastColId, selectNonPKColNum, selectNonPKColId))
+        return false;
    } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) {
-      if (hasLastFunc) {
+      SNode* pParam = nodesListGetNode(pAggFunc->pParameterList, 0);
+      if (QUERY_NODE_COLUMN == nodeType(pParam)) {
+        SColumnNode* pCol = (SColumnNode*)pParam;
+        if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) {
+          if (selectNonPKColId != pCol->colId) {
+            selectNonPKColId = pCol->colId;
+            selectNonPKColNum++;
+          }
+        } else {
+          continue;
+        }
+      } else if (lastColNum > 0) {
        return false;
      }
-      hasSelectFunc = true;
+      if (!lastRowScanOptCheckColNum(lastColNum, lastColId, selectNonPKColNum, selectNonPKColId))
+        return false;
    } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) {
      if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) {
        return false;
@ -2581,6 +2620,9 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic

  SLastRowScanOptSetColDataTypeCxt cxt = {.doAgg = true, .pLastCols = NULL};
  SNode*                           pNode = NULL;
+  SColumnNode*                     pPKTsCol = NULL;
+  SColumnNode*                     pNonPKCol = NULL;
+
  FOREACH(pNode, pAgg->pAggFuncs) {
    SFunctionNode* pFunc = (SFunctionNode*)pNode;
    int32_t        funcType = pFunc->funcType;
@ -2597,6 +2639,16 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
        nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt);
        nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1));
      }
+    } else if (FUNCTION_TYPE_SELECT_VALUE == funcType) {
+      pNode = nodesListGetNode(pFunc->pParameterList, 0);
+      if (nodeType(pNode) == QUERY_NODE_COLUMN) {
+        SColumnNode* pCol = (SColumnNode*)pNode;
+        if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+          pPKTsCol = pCol;
+        } else {
+          pNonPKCol = pCol;
+        }
+      }
    }
  }

@ -2608,6 +2660,16 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
    lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true);
    nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt);
    lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false);
+    if (pPKTsCol && pScan->node.pTargets->length == 1) {
+      // when select last(ts),ts from ..., we add another ts to targets
+      sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol);
+      nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pPKTsCol));
+    }
+    if (pNonPKCol && cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pNonPKCol, nodesListGetNode(cxt.pLastCols, 0))) {
+      // when select last(c1), c1 from ..., we add c1 to targets
+      sprintf(pNonPKCol->colName, "#sel_val.%p", pNonPKCol);
+      nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pNonPKCol));
+    }
    nodesClearList(cxt.pLastCols);
  }
  pAgg->hasLastRow = false;
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@ -552,6 +552,7 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu
  if (NULL == pScan) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }
+  pScan->pTargets = nodesCloneList(pScanLogicNode->node.pTargets);

  pScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags);
  if (NULL != pScanLogicNode->pGroupTags && NULL == pScan->pGroupTags) {
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@ -1230,7 +1230,6 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam
    code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128);
    if (code) {
      qError("func to_timestamp failed %s", errMsg);
-      code = code == -1 ? TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR : TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR;
      break;
    }
    colDataSetVal(pOutput->columnData, i, (char *)&ts, false);
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@ -106,7 +106,6 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
 int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq);

 int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId);
-int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
 int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet);

 int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId);
@ -127,13 +126,11 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask);
 int32_t streamTaskFillHistoryFinished(SStreamTask* pTask);
 int32_t streamTransferStateToStreamTask(SStreamTask* pTask);

-int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate);
+int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*);
 STaskId streamTaskExtractKey(const SStreamTask* pTask);
 void    streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo);
 void    streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo);

-void    streamMetaResetStartInfo(STaskStartInfo* pMeta);
-
 SStreamQueue* streamQueueOpen(int64_t cap);
 void          streamQueueClose(SStreamQueue* pQueue, int32_t taskId);
 void          streamQueueProcessSuccess(SStreamQueue* queue);
@ -142,6 +139,21 @@ void*         streamQueueNextItem(SStreamQueue* pQueue);
 void          streamFreeQitem(SStreamQueueItem* data);
 int32_t       streamQueueGetItemSize(const SStreamQueue* pQueue);

+typedef enum UPLOAD_TYPE{
+  UPLOAD_DISABLE = -1,
+  UPLOAD_S3 = 0,
+  UPLOAD_RSYNC = 1,
+} UPLOAD_TYPE;
+
+UPLOAD_TYPE getUploadType();
+int uploadCheckpoint(char* id, char* path);
+int downloadCheckpoint(char* id, char* path);
+int deleteCheckpoint(char* id);
+int deleteCheckpointFile(char* id, char* name);
+
+int32_t onNormalTaskReady(SStreamTask* pTask);
+int32_t onScanhistoryTaskReady(SStreamTask* pTask);
+
 #ifdef __cplusplus
 }
 #endif
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@ -14,6 +14,8 @@
 */

 #include "streamInt.h"
+#include "rsync.h"
+#include "cos.h"

 int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) {
  if (tStartEncode(pEncoder) < 0) return -1;
@ -372,3 +374,91 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {

  return code;
 }
+
+static int uploadCheckpointToS3(char* id, char* path){
+  TdDirPtr pDir = taosOpenDir(path);
+  if (pDir == NULL) return -1;
+
+  TdDirEntryPtr de = NULL;
+  while ((de = taosReadDir(pDir)) != NULL) {
+    char* name = taosGetDirEntryName(de);
+    if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 ||
+        taosDirEntryIsDir(de)) continue;
+
+    char filename[PATH_MAX] = {0};
+    if(path[strlen(path) - 1] == TD_DIRSEP_CHAR){
+      snprintf(filename, sizeof(filename), "%s%s", path, name);
+    }else{
+      snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name);
+    }
+
+    char object[PATH_MAX] = {0};
+    snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name);
+
+    if(s3PutObjectFromFile2(filename, object) != 0){
+      taosCloseDir(&pDir);
+      return -1;
+    }
+    stDebug("[s3] upload checkpoint:%s", filename);
+  }
+  taosCloseDir(&pDir);
+
+  return 0;
+}
+
+UPLOAD_TYPE getUploadType(){
+  if(strlen(tsSnodeAddress) != 0){
+    return UPLOAD_RSYNC;
+  }else if(tsS3StreamEnabled){
+    return UPLOAD_S3;
+  }else{
+    return UPLOAD_DISABLE;
+  }
+}
+
+int uploadCheckpoint(char* id, char* path){
+  if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){
+    stError("uploadCheckpoint parameters invalid");
+    return -1;
+  }
+  if(strlen(tsSnodeAddress) != 0){
+    return uploadRsync(id, path);
+  }else if(tsS3StreamEnabled){
+    return uploadCheckpointToS3(id, path);
+  }
+  return 0;
+}
+
+int downloadCheckpoint(char* id, char* path){
+  if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){
+    stError("downloadCheckpoint parameters invalid");
+    return -1;
+  }
+  if(strlen(tsSnodeAddress) != 0){
+    return downloadRsync(id, path);
+  }else if(tsS3StreamEnabled){
+    return s3GetObjectsByPrefix(id, path);
+  }
+  return 0;
+}
+
+int deleteCheckpoint(char* id){
+  if(id == NULL || strlen(id) == 0){
+    stError("deleteCheckpoint parameters invalid");
+    return -1;
+  }
+  if(strlen(tsSnodeAddress) != 0){
+    return deleteRsync(id);
+  }else if(tsS3StreamEnabled){
+    s3DeleteObjectsByPrefix(id);
+  }
+  return 0;
+}
+
+int deleteCheckpointFile(char* id, char* name){
+  char object[128] = {0};
+  snprintf(object, sizeof(object), "%s/%s", id, name);
+  char *tmp = object;
+  s3DeleteObjects((const char**)&tmp, 1);
+  return 0;
+}
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@ -1007,7 +1007,6 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo,
  info.msg.info = *pRpcInfo;

  taosThreadMutexLock(&pTask->lock);
-  stDebug("s-task:%s lock", pTask->id.idStr);

  if (pTask->pRspMsgList == NULL) {
    pTask->pRspMsgList = taosArrayInit(4, sizeof(SStreamContinueExecInfo));
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@ -18,7 +18,8 @@
 // maximum allowed processed block batches. One block may include several submit blocks
 #define MAX_STREAM_EXEC_BATCH_NUM         32
 #define STREAM_RESULT_DUMP_THRESHOLD      300
-#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1)
+#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1)   // 1MiB result data
+#define STREAM_SCAN_HISTORY_TIMESLICE     1000            // 1000 ms

 static int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask);

@ -48,10 +49,9 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl
    }

    streamDispatchStreamBlock(pTask);
-    return code;
  }

-  return 0;
+  return code;
 }

 static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* pRes, int32_t size, int64_t* totalSize,
@ -187,53 +187,47 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i
  return code;
 }

-int32_t streamScanHistoryData(SStreamTask* pTask) {
-  ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
+static int32_t handleResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (taosArrayGetSize(pRes) > 0) {
+    SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes);
+    code = doOutputResultBlockImpl(pTask, pStreamBlocks);
+    if (code != TSDB_CODE_SUCCESS) {
+      stDebug("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code));
+    }
+  } else {
+    taosArrayDestroy(pRes);
+  }
+  return code;
+}

+static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t* pSize, bool* pFinish) {
  int32_t code = TSDB_CODE_SUCCESS;
  void*   exec = pTask->exec.pExecutor;
-  bool    finished = false;
+  int32_t numOfBlocks = 0;

-  qSetStreamOpOpen(exec);
-
-  while (!finished) {
-    if (streamTaskShouldPause(pTask)) {
-      double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
-      stDebug("s-task:%s paused from the scan-history task, elapsed time:%.2fsec", pTask->id.idStr, el);
+  while (1) {
+    if (streamTaskShouldStop(pTask)) {
      break;
    }

-    SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
-    if (pRes == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-
-    int32_t size = 0;
-    int32_t numOfBlocks = 0;
-    while (1) {
-      if (streamTaskShouldStop(pTask)) {
-        taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
-        return 0;
-      }
-
    if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
-        stDebug("s-task:%s inputQ is blocked, wait for 10sec and retry", pTask->id.idStr);
-        taosMsleep(10000);
-        continue;
+      stDebug("s-task:%s level:%d inputQ is blocked, retry in 5s", pTask->id.idStr, pTask->info.taskLevel);
+      break;
    }

    SSDataBlock* output = NULL;
    uint64_t     ts = 0;
    code = qExecTask(exec, &output, &ts);
    if (code != TSDB_CODE_TSC_QUERY_KILLED && code != TSDB_CODE_SUCCESS) {
-        stError("%s scan-history data error occurred code:%s, continue scan", pTask->id.idStr, tstrerror(code));
+      stError("s-task:%s scan-history data error occurred code:%s, continue scan-history", pTask->id.idStr,
+              tstrerror(code));
      continue;
    }

    // the generated results before fill-history task been paused, should be dispatched to sink node
    if (output == NULL) {
-        finished = qStreamRecoverScanFinished(exec);
+      (*pFinish) = qStreamScanhistoryFinished(exec);
      break;
    }

@ -242,28 +236,69 @@ int32_t streamScanHistoryData(SStreamTask* pTask) {
    block.info.childId = pTask->info.selfChildId;
    taosArrayPush(pRes, &block);

-      size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
+    (*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
+    numOfBlocks += 1;

-      if ((++numOfBlocks) >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
+    if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || (*pSize) >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
      stDebug("s-task:%s scan exec numOfBlocks:%d, size:%.2fKiB output num-limit:%d, size-limit:%.2fKiB reached",
-                pTask->id.idStr, numOfBlocks, SIZE_IN_KiB(size), STREAM_RESULT_DUMP_THRESHOLD,
+              pTask->id.idStr, numOfBlocks, SIZE_IN_KiB(*pSize), STREAM_RESULT_DUMP_THRESHOLD,
              SIZE_IN_KiB(STREAM_RESULT_DUMP_SIZE_THRESHOLD));
      break;
    }
  }
-
-    if (taosArrayGetSize(pRes) > 0) {
-      SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes);
-      code = doOutputResultBlockImpl(pTask, pStreamBlocks);
-      if (code != TSDB_CODE_SUCCESS) {
-        return code;
-      }
-    } else {
-      taosArrayDestroy(pRes);
-    }
 }

-  return 0;
+SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
+  ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
+
+  void*   exec = pTask->exec.pExecutor;
+  bool    finished = false;
+
+  qSetStreamOpOpen(exec);
+
+  while (1) {
+    if (streamTaskShouldPause(pTask)) {
+      stDebug("s-task:%s paused from the scan-history task", pTask->id.idStr);
+      // quit from step1, not continue to handle the step2
+      return (SScanhistoryDataInfo){TASK_SCANHISTORY_QUIT, 0};
+    }
+
+    SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
+    if (pRes == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      stError("s-task:%s scan-history prepare result block failed, code:%s, retry later", pTask->id.idStr,
+              tstrerror(terrno));
+      continue;
+    }
+
+    int32_t size = 0;
+    streamScanHistoryDataImpl(pTask, pRes, &size, &finished);
+
+    if(streamTaskShouldStop(pTask)) {
+      taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
+      return (SScanhistoryDataInfo){TASK_SCANHISTORY_QUIT, 0};
+    }
+
+    // dispatch the generated results
+    int32_t code = handleResultBlocks(pTask, pRes, size);
+
+    int64_t el = taosGetTimestampMs() - st;
+
+    // downstream task input queue is full, try in 5sec
+    if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
+      return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 5000};
+    }
+
+    if (finished) {
+      return (SScanhistoryDataInfo){TASK_SCANHISTORY_CONT, 0};
+    }
+
+    if (el >= STREAM_SCAN_HISTORY_TIMESLICE) {
+      stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms",
+              pTask->id.idStr, pTask->info.fillHistory, el / 1000.0);
+      return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 100};
+    }
+  }
 }

 // wait for the stream task to be idle
@ -647,23 +682,25 @@ int32_t streamExecTask(SStreamTask* pTask) {
 int32_t streamTaskReleaseState(SStreamTask* pTask) {
  stDebug("s-task:%s release exec state", pTask->id.idStr);
  void* pExecutor = pTask->exec.pExecutor;
+
+  int32_t code = TSDB_CODE_SUCCESS;
  if (pExecutor != NULL) {
-    int32_t code = qStreamOperatorReleaseState(pExecutor);
-    return code;
-  } else {
-    return TSDB_CODE_SUCCESS;
+    code = qStreamOperatorReleaseState(pExecutor);
  }
+
+  return code;
 }

 int32_t streamTaskReloadState(SStreamTask* pTask) {
  stDebug("s-task:%s reload exec state", pTask->id.idStr);
  void* pExecutor = pTask->exec.pExecutor;
+
+  int32_t code = TSDB_CODE_SUCCESS;
  if (pExecutor != NULL) {
-    int32_t code = qStreamOperatorReloadState(pExecutor);
-    return code;
-  } else {
-    return TSDB_CODE_SUCCESS;
+    code = qStreamOperatorReloadState(pExecutor);
  }
+
+  return code;
 }

 int32_t streamAlignTransferState(SStreamTask* pTask) {
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@ -28,7 +28,6 @@ int32_t streamBackendId = 0;
 int32_t streamBackendCfWrapperId = 0;
 int32_t streamMetaId = 0;

-static int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta);
 static void    metaHbToMnode(void* param, void* tmrId);
 static void    streamMetaClear(SStreamMeta* pMeta);
 static int32_t streamMetaBegin(SStreamMeta* pMeta);
@ -150,6 +149,12 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF

  pMeta->startInfo.pReadyTaskSet = taosHashInit(64, fp, false, HASH_NO_LOCK);
  if (pMeta->startInfo.pReadyTaskSet == NULL) {
+    goto _err;
+  }
+
+  pMeta->startInfo.pFailedTaskSet = taosHashInit(4, fp, false, HASH_NO_LOCK);
+  if (pMeta->startInfo.pFailedTaskSet == NULL) {
+    goto _err;
  }

  pMeta->pHbInfo = taosMemoryCalloc(1, sizeof(SMetaHbInfo));
@ -185,10 +190,10 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
      taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
  pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t));
  pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t));
-  pMeta->chkpCap = 8;
+  pMeta->chkpCap = 2;
  taosInitRWLatch(&pMeta->chkpDirLock);

-  pMeta->chkpId = streamGetLatestCheckpointId(pMeta);
+  pMeta->chkpId = streamMetaGetLatestCheckpointId(pMeta);
  pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId);
  while (pMeta->streamBackend == NULL) {
    taosMsleep(100);
@ -221,6 +226,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
  if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo);
  if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks);
  if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
+  if (pMeta->startInfo.pFailedTaskSet) taosHashCleanup(pMeta->startInfo.pFailedTaskSet);
  taosMemoryFree(pMeta);

  stError("failed to open stream meta");
@ -228,12 +234,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
 }

 int32_t streamMetaReopen(SStreamMeta* pMeta) {
-  // backup the restart flag
-  int32_t restartFlag = pMeta->startInfo.startAllTasksFlag;
  streamMetaClear(pMeta);

-  pMeta->startInfo.startAllTasksFlag = restartFlag;
-
  // NOTE: role should not be changed during reopen meta
  pMeta->streamBackendRid = -1;
  pMeta->streamBackend = NULL;
@ -302,7 +304,10 @@ void streamMetaClear(SStreamMeta* pMeta) {
  pMeta->numOfPausedTasks = 0;
  pMeta->chkptNotReadyTasks = 0;

-  streamMetaResetStartInfo(&pMeta->startInfo);
+  // the willrestart/starting flag can NOT be cleared
+  taosHashClear(pMeta->startInfo.pReadyTaskSet);
+  taosHashClear(pMeta->startInfo.pFailedTaskSet);
+  pMeta->startInfo.readyTs = 0;
 }

 void streamMetaClose(SStreamMeta* pMeta) {
@ -342,6 +347,7 @@ void streamMetaCloseImpl(void* arg) {
  taosHashCleanup(pMeta->pTaskBackendUnique);
  taosHashCleanup(pMeta->updateInfo.pTasks);
  taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
+  taosHashCleanup(pMeta->startInfo.pFailedTaskSet);

  taosMemoryFree(pMeta->pHbInfo);
  taosMemoryFree(pMeta->path);
@ -595,7 +601,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) {
  return 0;
 }

-int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta) {
+int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) {
  int64_t chkpId = 0;

  TBC* pCur = NULL;
@ -1093,8 +1099,11 @@ void streamMetaInitForSnode(SStreamMeta* pMeta) {

 void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) {
  taosHashClear(pStartInfo->pReadyTaskSet);
-  pStartInfo->startAllTasksFlag = 0;
+  taosHashClear(pStartInfo->pFailedTaskSet);
+  pStartInfo->tasksWillRestart = 0;
  pStartInfo->readyTs = 0;
+  // reset the sentinel flag value to be 0
+  atomic_store_32(&pStartInfo->taskStarting, 0);
 }

 void streamMetaRLock(SStreamMeta* pMeta) {
@ -1104,7 +1113,6 @@ void streamMetaRLock(SStreamMeta* pMeta) {
 void streamMetaRUnLock(SStreamMeta* pMeta) {
  stTrace("vgId:%d meta-runlock", pMeta->vgId);
  taosRUnLockLatch(&pMeta->lock);
-
 }
 void streamMetaWLock(SStreamMeta* pMeta) {
  stTrace("vgId:%d meta-wlock", pMeta->vgId);
--- a/source/libs/stream/src/streamQueue.c
+++ b/source/libs/stream/src/streamQueue.c
@ -159,7 +159,8 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu

  // no available token in bucket for sink task, let's wait for a little bit
  if (taskLevel == TASK_LEVEL__SINK && (!streamTaskExtractAvailableToken(pTask->outputInfo.pTokenBucket, pTask->id.idStr))) {
-    stDebug("s-task:%s no available token in bucket for sink data, wait for 50ms", id);
+    stDebug("s-task:%s no available token in bucket for sink data, wait for 10ms", id);
+    taosMsleep(10);
    return TSDB_CODE_SUCCESS;
  }

@ -340,10 +341,11 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
  return 0;
 }

-// the result should be put into the outputQ in any cases, otherwise, the result may be lost
+// the result should be put into the outputQ in any cases, the result may be lost otherwise.
 int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock) {
  STaosQueue* pQueue = pTask->outputq.queue->pQueue;

+  // wait for the output queue is available for new data to dispatch
  while (streamQueueIsFull(pTask->outputq.queue)) {
    if (streamTaskShouldStop(pTask)) {
      stInfo("s-task:%s discard result block due to task stop", pTask->id.idStr);
@ -373,7 +375,8 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
  return TSDB_CODE_SUCCESS;
 }

-int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate) {
+int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate,
+                                  const char* id) {
  if (numCap < 10 || numRate < 10 || pBucket == NULL) {
    stError("failed to init sink task bucket, cap:%d, rate:%d", numCap, numRate);
    return TSDB_CODE_INVALID_PARA;
@ -388,6 +391,7 @@ int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t
  pBucket->quotaRemain = pBucket->quotaCapacity;

  pBucket->fillTimestamp = taosGetTimestampMs();
+  stDebug("s-task:%s sink quotaRate:%.2fMiB, numRate:%d", id, quotaRate, numRate);
  return TSDB_CODE_SUCCESS;
 }

@ -406,12 +410,12 @@ static void fillTokenBucket(STokenBucket* pBucket, const char* id) {
  double incSize = (delta / 1000.0) * pBucket->quotaRate;
  if (incSize > 0) {
    pBucket->quotaRemain = TMIN(pBucket->quotaRemain + incSize, pBucket->quotaCapacity);
+    pBucket->fillTimestamp = now;
  }

  if (incNum > 0 || incSize > 0) {
-    stDebug("new token and capacity available, current token:%d inc:%d, current quota:%.2fMiB inc:%.2fMiB, ts:%" PRId64
-            " idle for %.2f Sec, %s",
-            pBucket->numOfToken, incNum, pBucket->quotaRemain, incSize, now, delta / 1000.0, id);
+    stTrace("token/quota available, token:%d inc:%d, quota:%.2fMiB inc:%.3fMiB, ts:%" PRId64 " idle:%" PRId64 "ms, %s",
+            pBucket->numOfToken, incNum, pBucket->quotaRemain, incSize, now, delta, id);
  }
 }

--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@ -19,7 +19,6 @@
 #include "streamBackendRocksdb.h"
 #include "streamInt.h"
 #include "tcommon.h"
-#include "streamInt.h"

 enum SBackendFileType {
  ROCKSDB_OPTIONS_TYPE = 1,
@ -52,6 +51,7 @@ struct SStreamSnapHandle {
  int8_t         filetype;
  SArray*        pFileList;
  int32_t        currFileIdx;
+  int8_t         delFlag;  // 0 : not del, 1: del
 };
 struct SStreamSnapBlockHdr {
  int8_t  type;
@ -148,6 +148,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
      taosMemoryFree(tdir);
      return code;
    }
+    pHandle->delFlag = 1;
    chkpId = 0;
  }

@ -193,8 +194,8 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
      taosArrayPush(pFile->pSst, &sst);
    }
  }
-  {
-    char* buf = taosMemoryCalloc(1, 512);
+  if (qDebugFlag & DEBUG_TRACE) {
+    char* buf = taosMemoryCalloc(1, 128 + taosArrayGetSize(pFile->pSst) * 16);
    sprintf(buf, "[current: %s,", pFile->pCurrent);
    sprintf(buf + strlen(buf), "MANIFEST: %s,", pFile->pMainfest);
    sprintf(buf + strlen(buf), "options: %s,", pFile->pOptions);
@ -274,7 +275,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
  if (handle->checkpointId == 0) {
    // del tmp dir
    if (pFile && taosIsDir(pFile->path)) {
-      taosRemoveDir(pFile->path);
+      if (handle->delFlag) taosRemoveDir(pFile->path);
    }
  } else {
    streamBackendDelInUseChkp(handle->handle, handle->checkpointId);
@ -423,6 +424,7 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path
  pHandle->pFileList = list;
  pHandle->currFileIdx = 0;
  pHandle->offset = 0;
+  pHandle->delFlag = 0;

  *ppWriter = pWriter;
  return 0;
@ -480,8 +482,8 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
 }
 int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) {
  SStreamSnapHandle* handle = &pWriter->handle;
-  if (qDebugFlag & DEBUG_DEBUG) {
-    char* buf = (char*)taosMemoryMalloc(1024);
+  if (qDebugFlag & DEBUG_TRACE) {
+    char* buf = (char*)taosMemoryMalloc(128 + taosArrayGetSize(handle->pFileList) * 16);
    int   n = sprintf(buf, "[");
    for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) {
      SBackendFileItem* item = taosArrayGet(handle->pFileList, i);
--- a/source/libs/stream/src/streamStart.c
+++ b/source/libs/stream/src/streamStart.c
@ -19,6 +19,10 @@
 #include "wal.h"
 #include "streamsm.h"

+#define SCANHISTORY_IDLE_TIME_SLICE 100    // 100ms
+#define SCANHISTORY_MAX_IDLE_TIME   10     // 10 sec
+#define SCANHISTORY_IDLE_TICK      ((SCANHISTORY_MAX_IDLE_TIME * 1000) / SCANHISTORY_IDLE_TIME_SLICE)
+
 typedef struct SLaunchHTaskInfo {
  SStreamMeta* pMeta;
  STaskId      id;
@ -30,6 +34,12 @@ typedef struct STaskRecheckInfo {
  void*               checkTimer;
 } STaskRecheckInfo;

+typedef struct STaskInitTs {
+  int64_t start;
+  int64_t end;
+  bool    success;
+} STaskInitTs;
+
 static int32_t           streamSetParamForScanHistory(SStreamTask* pTask);
 static void              streamTaskSetRangeStreamCalc(SStreamTask* pTask);
 static int32_t           initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated);
@ -57,7 +67,7 @@ int32_t streamTaskSetReady(SStreamTask* pTask) {
  stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s",
          pTask->id.idStr, numOfDowns, el, p);

-  streamMetaUpdateTaskReadyInfo(pTask);
+  streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true);
  return TSDB_CODE_SUCCESS;
 }

@ -81,6 +91,60 @@ int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) {
  return 0;
 }

+static void doReExecScanhistory(void* param, void* tmrId) {
+  SStreamTask* pTask = param;
+  pTask->schedHistoryInfo.numOfTicks -= 1;
+
+  char* p = NULL;
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
+  if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) {
+    streamMetaReleaseTask(pTask->pMeta, pTask);
+    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p, ref);
+    return;
+  }
+
+  if (pTask->schedHistoryInfo.numOfTicks <= 0) {
+    streamStartScanHistoryAsync(pTask, 0);
+
+    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
+    stDebug("s-task:%s fill-history:%d start scan-history data, out of tmr, ref:%d", pTask->id.idStr,
+            pTask->info.fillHistory, ref);
+
+    // release the task.
+    streamMetaReleaseTask(pTask->pMeta, pTask);
+  } else {
+    taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer,
+                 &pTask->schedHistoryInfo.pTimer);
+  }
+}
+
+int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration) {
+  int32_t numOfTicks = idleDuration / SCANHISTORY_IDLE_TIME_SLICE;
+  if (numOfTicks <= 0) {
+    numOfTicks = 1;
+  } else if (numOfTicks > SCANHISTORY_IDLE_TICK) {
+    numOfTicks = SCANHISTORY_IDLE_TICK;
+  }
+
+  // add ref for task
+  SStreamTask* p = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId);
+  ASSERT(p != NULL);
+
+  pTask->schedHistoryInfo.numOfTicks = numOfTicks;
+
+  int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
+  stDebug("s-task:%s scan-history start in %.2fs, ref:%d", pTask->id.idStr, numOfTicks*0.1, ref);
+
+  if (pTask->schedHistoryInfo.pTimer == NULL) {
+    pTask->schedHistoryInfo.pTimer = taosTmrStart(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer);
+  } else {
+    taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer, &pTask->schedHistoryInfo.pTimer);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t doStartScanHistoryTask(SStreamTask* pTask) {
  SVersionRange* pRange = &pTask->dataRange.range;
  if (pTask->info.fillHistory) {
@ -318,6 +382,31 @@ void doProcessDownstreamReadyRsp(SStreamTask* pTask) {
  streamTaskOnHandleEventSuccess(pTask->status.pSM, event);
 }

+static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
+  int32_t vgId = pTask->pMeta->vgId;
+
+  taosThreadMutexLock(&pTask->lock);
+  int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList);
+  bool    existed = false;
+  for (int i = 0; i < num; ++i) {
+    SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, i);
+    if (p->nodeId == nodeId) {
+      existed = true;
+      break;
+    }
+  }
+
+  if (!existed) {
+    SDownstreamTaskEpset t = {.nodeId = nodeId};
+    taosArrayPush(pTask->outputInfo.pDownstreamUpdateList, &t);
+
+    stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId,
+           t.nodeId, (int32_t)taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList));
+  }
+
+  taosThreadMutexUnlock(&pTask->lock);
+}
+
 int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
  ASSERT(pTask->id.taskId == pRsp->upstreamTaskId);
  const char* id = pTask->id.idStr;
@ -367,40 +456,23 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs
      doProcessDownstreamReadyRsp(pTask);
    }
  } else {  // not ready, wait for 100ms and retry
+    if (pRsp->status == TASK_UPSTREAM_NEW_STAGE || pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) {
      if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) {
        stError(
            "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, "
            "not check wait for downstream task nodeUpdate, and all tasks restart",
            id, pRsp->upstreamNodeId, pRsp->oldStage, (int32_t)pTask->pMeta->stage);
      } else {
-      if (pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) {
        stError(
            "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check "
            "downstream again, nodeUpdate needed",
            id, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
-
-        taosThreadMutexLock(&pTask->lock);
-        int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList);
-        bool existed = false;
-        for (int i = 0; i < num; ++i) {
-          SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, i);
-          if (p->nodeId == pRsp->downstreamNodeId) {
-            existed = true;
-            break;
-          }
      }

-        if (!existed) {
-          SDownstreamTaskEpset t = {.nodeId = pRsp->downstreamNodeId};
-          taosArrayPush(pTask->outputInfo.pDownstreamUpdateList, &t);
-          stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", id, vgId,
-                 t.nodeId, (int32_t)taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList));
-        }
-
-        taosThreadMutexUnlock(&pTask->lock);
-        return 0;
-      }
+      addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId);
+      streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, taosGetTimestampMs(), false);

+    } else {  // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms
      STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp);

      int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
@ -676,8 +748,7 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) {
        int32_t hTaskId = pHTaskInfo->id.taskId;

        streamTaskGetStatus(pTask, &p);
-        stDebug(
-            "s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d",
+        stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d",
                pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes);

        taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamEnv.timer, &pHTaskInfo->pTimer);
@ -975,28 +1046,57 @@ void streamTaskEnablePause(SStreamTask* pTask) {
  pTask->status.pauseAllowed = 1;
 }

-int32_t streamMetaUpdateTaskReadyInfo(SStreamTask* pTask) {
+static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ) {
+  int32_t vgId = pMeta->vgId;
+  void*   pIter = NULL;
+  size_t  keyLen = 0;
+
+  stInfo("vgId:%d %d tasks check-downstream completed %s", vgId, taosHashGetSize(pTaskSet),
+         succ ? "success" : "failed");
+
+  while ((pIter = taosHashIterate(pTaskSet, pIter)) != NULL) {
+    STaskInitTs* pInfo = pIter;
+    void*        key = taosHashGetKey(pIter, &keyLen);
+
+    SStreamTask** pTask1 = taosHashGet(pMeta->pTasksMap, key, sizeof(STaskId));
+    if (pTask1 == NULL) {
+      stInfo("s-task:0x%x is dropped already, %s", (int32_t)((STaskId*)key)->taskId, succ ? "success" : "failed");
+    } else {
+      stInfo("s-task:%s level:%d vgId:%d, init:%" PRId64 ", initEnd:%" PRId64 ", %s", (*pTask1)->id.idStr,
+             (*pTask1)->info.taskLevel, vgId, pInfo->start, pInfo->end, pInfo->success ? "success" : "failed");
+    }
+  }
+}
+
+int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool ready) {
  SStreamMeta* pMeta = pTask->pMeta;

  streamMetaWLock(pMeta);

  STaskId id = streamTaskExtractKey(pTask);
-  taosHashPut(pMeta->startInfo.pReadyTaskSet, &id, sizeof(id), NULL, 0);
+  STaskStartInfo* pStartInfo = &pMeta->startInfo;
+
+  SHashObj* pDst = ready? pStartInfo->pReadyTaskSet:pStartInfo->pFailedTaskSet;
+
+  STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready};
+  taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs));

  int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta);

-  if (taosHashGetSize(pMeta->startInfo.pReadyTaskSet) == numOfTotal) {
-    STaskStartInfo* pStartInfo = &pMeta->startInfo;
-
+  if (taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet) == numOfTotal) {
    pStartInfo->readyTs = pTask->execInfo.start;
    pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0;

-    streamMetaResetStartInfo(pStartInfo);
-
-    stDebug("vgId:%d all %d task(s) are started successfully, last ready task:%s level:%d, startTs:%" PRId64
+    stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:%s level:%d, startTs:%" PRId64
            ", readyTs:%" PRId64 " total elapsed time:%.2fs",
            pMeta->vgId, numOfTotal, pTask->id.idStr, pTask->info.taskLevel, pStartInfo->startTs, pStartInfo->readyTs,
            pStartInfo->elapsedTime / 1000.0);
+
+    // print the initialization elapsed time and info
+    displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true);
+    displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false);
+
+    streamMetaResetStartInfo(pStartInfo);
  }

  streamMetaWUnLock(pMeta);
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@ -221,7 +221,6 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
  }

  pState->pTdbState->pOwner = pTask;
-  pState->checkPointId = 0;

  return pState;

@ -274,7 +273,6 @@ int32_t streamStateCommit(SStreamState* pState) {
    SStreamSnapshot* pShot = getSnapshot(pState->pFileState);
    flushSnapshot(pState->pFileState, pShot, true);
  }
-  pState->checkPointId++;
  return 0;
 #else
  if (tdbCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) {
@ -288,7 +286,6 @@ int32_t streamStateCommit(SStreamState* pState) {
               TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
    return -1;
  }
-  pState->checkPointId++;
  return 0;
 #endif
 }
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@ -400,8 +400,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
  taosMemoryFree(pTask->outputInfo.pTokenBucket);
  taosThreadMutexDestroy(&pTask->lock);

-  taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList);
-  pTask->outputInfo.pDownstreamUpdateList = NULL;
+  pTask->outputInfo.pDownstreamUpdateList = taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList);

  taosMemoryFree(pTask);
  stDebug("s-task:0x%x free task completed", taskId);
@ -447,7 +446,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i

  // 2MiB per second for sink task
  // 50 times sink operator per second
-  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate);
+  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate, pTask->id.idStr);

  TdThreadMutexAttr attr = {0};
  int code = taosThreadMutexAttrInit(&attr);
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@ -315,7 +315,6 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
            GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt));

    taosThreadMutexUnlock(&pTask->lock);
-    stDebug("s-task:%s unlockx", pTask->id.idStr);
    return TSDB_CODE_STREAM_INVALID_STATETRANS;
  }

--- a/source/libs/stream/src/tstreamFileState.c
+++ b/source/libs/stream/src/tstreamFileState.c
@ -27,6 +27,9 @@
 #define DEFAULT_MAX_STREAM_BUFFER_SIZE (128 * 1024 * 1024)
 #define MIN_NUM_OF_ROW_BUFF            10240

+#define TASK_KEY                       "streamFileState"
+#define STREAM_STATE_INFO_NAME         "StreamStateCheckPoint"
+
 struct SStreamFileState {
  SList*   usedBuffs;
  SList*   freeBuffs;
@ -113,6 +116,15 @@ void* sessionCreateStateKey(SRowBuffPos* pPos, int64_t num) {
  return pStateKey;
 }

+static void streamFileStateDecode(TSKEY* pKey, void* pBuff, int32_t len) { pBuff = taosDecodeFixedI64(pBuff, pKey); }
+
+static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) {
+  *pLen = sizeof(TSKEY);
+  (*pVal) = taosMemoryCalloc(1, *pLen);
+  void* buff = *pVal;
+  taosEncodeFixedI64(&buff, *pKey);
+}
+
 SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
                                      GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId,
                                      int8_t type) {
@ -181,6 +193,15 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_
    recoverSesssion(pFileState, checkpointId);
  }

+  void*   valBuf = NULL;
+  int32_t len = 0;
+  int32_t code = streamDefaultGet_rocksdb(pFileState->pFileStore, STREAM_STATE_INFO_NAME, &valBuf, &len);
+  if (code == TSDB_CODE_SUCCESS) {
+    ASSERT(len == sizeof(TSKEY));
+    streamFileStateDecode(&pFileState->flushMark, valBuf, len);
+    qDebug("===stream===flushMark  read:%" PRId64, pFileState->flushMark);
+  }
+  taosMemoryFreeClear(valBuf);
  return pFileState;

 _error:
@ -506,15 +527,6 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) {
  return pFileState->usedBuffs;
 }

-void streamFileStateDecode(TSKEY* pKey, void* pBuff, int32_t len) { pBuff = taosDecodeFixedI64(pBuff, pKey); }
-
-void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) {
-  *pLen = sizeof(TSKEY);
-  (*pVal) = taosMemoryCalloc(1, *pLen);
-  void* buff = *pVal;
-  taosEncodeFixedI64(&buff, *pKey);
-}
-
 int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) {
  int32_t   code = TSDB_CODE_SUCCESS;
  SListIter iter = {0};
@ -538,6 +550,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
      continue;
    }
    pPos->beFlushed = true;
+    pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey));

    qDebug("===stream===flushed start:%" PRId64, pFileState->getTs(pPos->pKey));
    if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) {
@ -565,24 +578,12 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
         pFileState->id, numOfElems, BATCH_LIMIT, elapsed);

  if (flushState) {
-    const char* taskKey = "streamFileState";
-    {
-      char    keyBuf[128] = {0};
    void*   valBuf = NULL;
    int32_t len = 0;
-      sprintf(keyBuf, "%s:%" PRId64 "", taskKey, ((SStreamState*)pFileState->pFileStore)->checkPointId);
    streamFileStateEncode(&pFileState->flushMark, &valBuf, &len);
-      streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0);
+    qDebug("===stream===flushMark write:%" PRId64, pFileState->flushMark);
+    streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0);
    taosMemoryFree(valBuf);
-    }
-    {
-      char    keyBuf[128] = {0};
-      char    valBuf[64] = {0};
-      int32_t len = 0;
-      memcpy(keyBuf, taskKey, strlen(taskKey));
-      len = sprintf(valBuf, "%" PRId64 "", ((SStreamState*)pFileState->pFileStore)->checkPointId);
-      code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0);
-    }
    streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
  }

@ -591,26 +592,23 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
 }

 int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) {
-  const char* taskKey = "streamFileState";
  char        keyBuf[128] = {0};
-  sprintf(keyBuf, "%s:%" PRId64 "", taskKey, checkpointId);
+  sprintf(keyBuf, "%s:%" PRId64 "", TASK_KEY, checkpointId);
  return streamDefaultDel_rocksdb(pFileState->pFileStore, keyBuf);
 }

 int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list) {
-  const char* taskKey = "streamFileState";
-  return streamDefaultIterGet_rocksdb(pFileState->pFileStore, taskKey, NULL, list);
+  return streamDefaultIterGet_rocksdb(pFileState->pFileStore, TASK_KEY, NULL, list);
 }

 int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
  int32_t     code = TSDB_CODE_SUCCESS;
-  const char* taskKey = "streamFileState";
  int64_t     maxCheckPointId = 0;
  {
    char    buf[128] = {0};
    void*   val = NULL;
    int32_t len = 0;
-    memcpy(buf, taskKey, strlen(taskKey));
+    memcpy(buf, TASK_KEY, strlen(TASK_KEY));
    code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len);
    if (code != 0 || len == 0 || val == NULL) {
      return TSDB_CODE_FAILED;
@ -624,7 +622,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
    char    buf[128] = {0};
    void*   val = 0;
    int32_t len = 0;
-    sprintf(buf, "%s:%" PRId64 "", taskKey, i);
+    sprintf(buf, "%s:%" PRId64 "", TASK_KEY, i);
    code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len);
    if (code != 0) {
      return TSDB_CODE_FAILED;
--- a/source/libs/stream/test/CMakeLists.txt
+++ b/source/libs/stream/test/CMakeLists.txt
@ -18,6 +18,17 @@ TARGET_INCLUDE_DIRECTORIES(
  PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
 )

+ADD_EXECUTABLE(checkpointTest checkpointTest.cpp)
+TARGET_LINK_LIBRARIES(
+        checkpointTest
+        PUBLIC os common gtest stream executor qcom index transport util
+)
+
+TARGET_INCLUDE_DIRECTORIES(
+        checkpointTest
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+)
+
 add_test(
  NAME streamUpdateTest
  COMMAND streamUpdateTest
--- a/source/libs/stream/test/checkpointTest.cpp
+++ b/source/libs/stream/test/checkpointTest.cpp
@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include <taoserror.h>
+#include <tglobal.h>
+#include <iostream>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#include "rsync.h"
+#include "streamInt.h"
+#include "cos.h"
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+
+  if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) {
+    printf("error");
+  }
+  if (s3Init() < 0) {
+    return -1;
+  }
+  strcpy(tsSnodeAddress, "127.0.0.1");
+  return RUN_ALL_TESTS();
+}
+
+TEST(testCase, checkpointUpload_Test) {
+  stopRsync();
+  startRsync();
+
+  taosSsleep(5);
+  char* id = "2013892036";
+
+  uploadCheckpoint(id, "/root/offset/");
+}
+
+TEST(testCase, checkpointDownload_Test) {
+  char* id = "2013892036";
+  downloadCheckpoint(id, "/root/offset/download/");
+}
+
+TEST(testCase, checkpointDelete_Test) {
+  char* id = "2013892036";
+  deleteCheckpoint(id);
+}
+
+TEST(testCase, checkpointDeleteFile_Test) {
+  char* id = "2013892036";
+  deleteCheckpointFile(id, "offset-ver0");
+}
--- a/source/libs/sync/src/syncSnapshot.c
+++ b/source/libs/sync/src/syncSnapshot.c
@ -865,6 +865,9 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot
  ASSERT(pRcvBuf->start <= pRcvBuf->cursor + 1 && pRcvBuf->cursor < pRcvBuf->end);

  if (pMsg->seq > pRcvBuf->cursor) {
+    if (pRcvBuf->entries[pMsg->seq % pRcvBuf->size]) {
+      pRcvBuf->entryDeleteCb(pRcvBuf->entries[pMsg->seq % pRcvBuf->size]);
+    }
    pRcvBuf->entries[pMsg->seq % pRcvBuf->size] = pMsg;
    ppMsg[0] = NULL;
    pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end);
@ -1002,7 +1005,8 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
  }

  if (pMsg->term < raftStoreGetTerm(pSyncNode)) {
-    syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject since small term");
+    sRError(pReceiver, "reject snap replication with smaller term. msg term:%" PRId64 ", seq:%d", pMsg->term,
+            pMsg->seq);
    terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
    return -1;
  }
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@ -268,7 +268,7 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla

  taosPrintLog(flags, level, dflag,
               "vgId:%d, %s, sync:%s, snap-sender:%p signature:(%" PRId64 ", %" PRId64 "), {start:%" PRId64
-               " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64
+               " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRId64 " last-cfg:%" PRId64
               ", seq:%d, ack:%d, "
               " buf:[%" PRId64 " %" PRId64 ", %" PRId64
               "), finish:%d, as:%d, to-dnode:%d}"
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@ -310,6 +310,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy
  GRANT_CFG_SET;
  SConfigItem *pItem = cfgGetItem(pCfg, name);
  if (pItem == NULL) {
+    terrno = TSDB_CODE_CFG_NOT_FOUND;
    return -1;
  }

@ -338,6 +339,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy
      break;
  }

+_err_out:
  terrno = TSDB_CODE_INVALID_CFG;
  return -1;
 }
@ -357,6 +359,59 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) {
  return NULL;
 }

+int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) {
+  ECfgDynType  dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT;
+  SConfigItem *pItem = cfgGetItem(pCfg, name);
+  if (!pItem || (pItem->dynScope & dynType) == 0) {
+    uError("failed to config:%s, not support", name);
+    terrno = TSDB_CODE_INVALID_CFG;
+    return -1;
+  }
+
+  switch (pItem->dtype) {
+    case CFG_DTYPE_BOOL: {
+      int32_t ival = (int32_t)atoi(pVal);
+      if (ival != 0 && ival != 1) {
+        uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name,
+               cfgDtypeStr(pItem->dtype), ival);
+        terrno = TSDB_CODE_OUT_OF_RANGE;
+        return -1;
+      }
+    } break;
+    case CFG_DTYPE_INT32: {
+      int32_t ival = (int32_t)atoi(pVal);
+      if (ival < pItem->imin || ival > pItem->imax) {
+        uError("cfg:%s, type:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
+               cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax);
+        terrno = TSDB_CODE_OUT_OF_RANGE;
+        return -1;
+      }
+    } break;
+    case CFG_DTYPE_INT64: {
+      int64_t ival = (int64_t)atoll(pVal);
+      if (ival < pItem->imin || ival > pItem->imax) {
+        uError("cfg:%s, type:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
+               cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax);
+        terrno = TSDB_CODE_OUT_OF_RANGE;
+        return -1;
+      }
+    } break;
+    case CFG_DTYPE_FLOAT:
+    case CFG_DTYPE_DOUBLE: {
+      float fval = (float)atof(pVal);
+      if (fval < pItem->fmin || fval > pItem->fmax) {
+        uError("cfg:%s, type:%s value:%f out of range[%f, %f]", pItem->name, cfgDtypeStr(pItem->dtype), fval,
+               pItem->fmin, pItem->fmax);
+        terrno = TSDB_CODE_OUT_OF_RANGE;
+        return -1;
+      }
+    } break;
+    default:
+      break;
+  }
+  return 0;
+}
+
 static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
  pItem->stype = CFG_STYPE_DEFAULT;
  pItem->name = taosStrdup(name);
@ -381,43 +436,61 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
  return 0;
 }

-int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope) {
-  SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope};
+int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope) {
+  SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope, .dynScope = dynScope};
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) {
+int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope,
+                    int8_t dynScope) {
  if (defaultVal < minval || defaultVal > maxval) {
    terrno = TSDB_CODE_OUT_OF_RANGE;
    return -1;
  }

-  SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .scope = scope};
+  SConfigItem item = {.dtype = CFG_DTYPE_INT32,
+                      .i32 = defaultVal,
+                      .imin = minval,
+                      .imax = maxval,
+                      .scope = scope,
+                      .dynScope = dynScope};
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) {
+int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope,
+                    int8_t dynScope) {
  if (defaultVal < minval || defaultVal > maxval) {
    terrno = TSDB_CODE_OUT_OF_RANGE;
    return -1;
  }

-  SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .scope = scope};
+  SConfigItem item = {.dtype = CFG_DTYPE_INT64,
+                      .i64 = defaultVal,
+                      .imin = minval,
+                      .imax = maxval,
+                      .scope = scope,
+                      .dynScope = dynScope};
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope) {
+int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope,
+                    int8_t dynScope) {
  if (defaultVal < minval || defaultVal > maxval) {
    terrno = TSDB_CODE_OUT_OF_RANGE;
    return -1;
  }

-  SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .scope = scope};
+  SConfigItem item = {.dtype = CFG_DTYPE_FLOAT,
+                      .fval = defaultVal,
+                      .fmin = minval,
+                      .fmax = maxval,
+                      .scope = scope,
+                      .dynScope = dynScope};
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
-  SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope};
+int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
+  SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope};
  item.str = taosStrdup(defaultVal);
  if (item.str == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -426,8 +499,8 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
-  SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope};
+int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
+  SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope};
  if (cfgCheckAndSetDir(&item, defaultVal) != 0) {
    return -1;
  }
@ -435,8 +508,8 @@ int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
-  SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope};
+int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
+  SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope};
  if (cfgCheckAndSetLocale(&item, defaultVal) != 0) {
    return -1;
  }
@ -444,8 +517,8 @@ int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, in
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
-  SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope};
+int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
+  SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope};
  if (cfgCheckAndSetCharset(&item, defaultVal) != 0) {
    return -1;
  }
@ -453,8 +526,8 @@ int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, i
  return cfgAddItem(pCfg, &item, name);
 }

-int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) {
-  SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope};
+int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) {
+  SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope};
  if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) {
    return -1;
  }
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@ -613,8 +613,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE,        "Invalid function par
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE,       "Invalid function para value")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION,      "Not buildin function")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP,            "Duplicate timestamps not allowed in function")
-TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed, format mismatch")
-TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed, wrong timestamp")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed for format mismatch")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format")

 //udf
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING,                   "udf is stopping")
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@ -57,7 +57,6 @@ typedef struct {

 typedef struct {
  int32_t       fileNum;
-  int32_t       maxLines;
  int32_t       lines;
  int32_t       flag;
  int32_t       openInProgress;
@ -122,7 +121,7 @@ static void     *taosAsyncOutputLog(void *param);
 static int32_t   taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msgLen);
 static SLogBuff *taosLogBuffNew(int32_t bufSize);
 static void      taosCloseLogByFd(TdFilePtr pFile);
-static int32_t   taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum);
+static int32_t   taosOpenLogFile(char *fn, int32_t maxFileNum);

 static FORCE_INLINE void taosUpdateDaylight() {
  struct tm      Tm, *ptm;
@ -186,7 +185,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) {

  tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE);
  if (tsLogObj.logHandle == NULL) return -1;
-  if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1;
+  if (taosOpenLogFile(fullName, maxFiles) < 0) return -1;

  if (taosInitSlowLog() < 0) return -1;
  if (taosStartLog() < 0) return -1;
@ -283,7 +282,7 @@ static void *taosThreadToOpenNewFile(void *param) {
  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
  if (pFile == NULL) {
    tsLogObj.openInProgress = 0;
-    tsLogObj.lines = tsLogObj.maxLines - 1000;
+    tsLogObj.lines = tsNumOfLogLines - 1000;
    uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno));
    return NULL;
  }
@ -308,7 +307,7 @@ static void *taosThreadToOpenNewFile(void *param) {
 static int32_t taosOpenNewLogFile() {
  taosThreadMutexLock(&tsLogObj.logMutex);

-  if (tsLogObj.lines > tsLogObj.maxLines && tsLogObj.openInProgress == 0) {
+  if (tsLogObj.lines > tsNumOfLogLines && tsLogObj.openInProgress == 0) {
    tsLogObj.openInProgress = 1;

    uInfo("open new log file ......");
@ -331,7 +330,7 @@ void taosResetLog() {
  sprintf(lastName, "%s.%d", tsLogObj.logName, tsLogObj.flag);

  // force create a new log file
-  tsLogObj.lines = tsLogObj.maxLines + 10;
+  tsLogObj.lines = tsNumOfLogLines + 10;

  taosOpenNewLogFile();
  (void)taosRemoveFile(lastName);
@ -384,7 +383,7 @@ static void taosGetLogFileName(char *fn) {
  }
 }

-static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
+static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) {
 #ifdef WINDOWS_STASH
  /*
   * always set maxFileNum to 1
@ -396,7 +395,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
  char    name[LOG_FILE_NAME_LEN + 50] = "\0";
  int32_t logstat0_mtime, logstat1_mtime;

-  tsLogObj.maxLines = maxLines;
  tsLogObj.fileNum = maxFileNum;
  taosGetLogFileName(fn);

@ -497,9 +495,9 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
      taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
    }

-    if (tsLogObj.maxLines > 0) {
+    if (tsNumOfLogLines > 0) {
      atomic_add_fetch_32(&tsLogObj.lines, 1);
-      if ((tsLogObj.lines > tsLogObj.maxLines) && (tsLogObj.openInProgress == 0)) {
+      if ((tsLogObj.lines > tsNumOfLogLines) && (tsLogObj.openInProgress == 0)) {
        taosOpenNewLogFile();
      }
    }
--- a/Show More
+++ b/Show More