Merge branch '3.0' into fix/TD-26189-last-tier-write

2023-11-01 10:53:55 +08:00 · 2023-11-01 10:53:55 +08:00 · 64644d673b
parent a61502411d 7359f98eca
commit 64644d673b
123 changed files with 9819 additions and 7644 deletions
--- a/2
+++ b/2
@ -424,7 +424,7 @@ pipeline {
                            echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
                        }
                        catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                            timeout(time: 130, unit: 'MINUTES'){
+                            timeout(time: 150, unit: 'MINUTES'){
                                pre_test()
                                script {
                                    sh '''
--- a/deps/arm/dm_static/libdmodule.a
+++ b/deps/arm/dm_static/libdmodule.a
--- a/deps/darwin/arm/dm_static/libdmodule.a
+++ b/deps/darwin/arm/dm_static/libdmodule.a
--- a/deps/darwin/x64/dm_static/libdmodule.a
+++ b/deps/darwin/x64/dm_static/libdmodule.a
--- a/deps/win/x64/dm_static/dmodule.lib
+++ b/deps/win/x64/dm_static/dmodule.lib
--- a/deps/x86/dm_static/libdmodule.a
+++ b/deps/x86/dm_static/libdmodule.a
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@ -483,6 +483,91 @@ return_timestamp: {
 - The precision of the returned timestamp is same as the precision set for the current data base in use
 - return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT.

+#### TO_CHAR
+
+```sql
+TO_CHAR(ts, format_str_literal)
+```
+
+**Description**: Convert a ts column to string as the format specified
+
+**Return value type**: VARCHAR
+
+**Applicable column types**: TIMESTAMP
+
+**Nested query**: It can be used in both the outer query and inner query in a nested query.
+
+**Applicable table types**: standard tables and supertables
+
+**Supported Formats**
+
+| **Format** | **Comment**| **example** |
+| --- | --- | --- |
+|AM,am,PM,pm| Meridiem indicator(without periods) | 07:00:00am|
+|A.M.,a.m.,P.M.,p.m.| Meridiem indicator(with periods)| 07:00:00a.m.|
+|YYYY,yyyy|year, 4 or more digits| 2023-10-10|
+|YYY,yyy| year, last 3 digits| 023-10-10|
+|YY,yy| year, last 2 digits| 23-10-10|
+|Y,y| year, last digit| 3-10-10|
+|MONTH|full uppercase of month| 2023-JANUARY-01|
+|Month|full capitalized month| 2023-January-01|
+|month|full lowercase of month| 2023-january-01|
+|MON| abbreviated uppercase of month(3 char)| JAN, SEP|
+|Mon| abbreviated capitalized month| Jan, Sep|
+|mon|abbreviated lowercase of month| jan, sep|
+|MM,mm|month number 01-12|2023-01-01|
+|DD,dd|month day, 01-31||
+|DAY|full uppercase of week day|MONDAY|
+|Day|full capitalized week day|Monday|
+|day|full lowercase of week day|monday|
+|DY|abbreviated uppercase of week day|MON|
+|Dy|abbreviated capitalized week day|Mon|
+|dy|abbreviated lowercase of week day|mon|
+|DDD|year day, 001-366||
+|D,d|week day number, 1-7, Sunday(1) to Saturday(7)||
+|HH24,hh24|hour of day, 00-23|2023-01-30 23:59:59|
+|hh12,HH12, hh, HH| hour of day, 01-12|2023-01-30 12:59:59PM|
+|MI,mi|minute, 00-59||
+|SS,ss|second, 00-59||
+|MS,ms|milli second, 000-999||
+|US,us|micro second, 000000-999999||
+|NS,ns|nano second, 000000000-999999999||
+|TZH,tzh|time zone hour|2023-01-30 11:59:59PM +08|
+
+**More explanations**:
+- The output format of `Month`, `Day` are left aligined, like`2023-OCTOBER  -01`, `2023-SEPTEMBER-01`, `September` is the longest, no paddings. Week days are slimilar.
+- When `ms`,`us`,`ns` are used in `to_char`, like `to_char(ts, 'yyyy-mm-dd hh:mi:ss.ms.us.ns')`, The time of `ms`,`us`,`ns` corresponds to the same fraction seconds. When ts is `1697182085123`, the output of `ms` is `123`, `us` is `123000`, `ns` is `123000000`.
+- If we want to output some characters of format without converting, surround it with double quotes. `to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. If want to output double quotes, add a back slash before double quote, like `to_char(ts, '\"yyyy-mm-dd\"')` will output `"2023-10-10"`.
+- For formats that output digits, the uppercase and lowercase formats are the same.
+- It's recommended to put time zone in the format, if not, the default time zone zone will be that in server or client.
+
+#### TO_TIMESTAMP
+
+```sql
+TO_TIMESTAMP(ts_str_literal, format_str_literal)
+```
+
+**Description**: Convert a formated timestamp string to a timestamp
+
+**Return value type**: TIMESTAMP
+
+**Applicable column types**: VARCHAR
+
+**Nested query**: It can be used in both the outer query and inner query in a nested query.
+
+**Applicable table types**: standard tables and supertables
+
+**Supported Formats**: The same as `TO_CHAR`.
+
+**More explanations**:
+- When `ms`, `us`, `ns` are used in `to_timestamp`, if multi of them are specified, the results are accumulated. For example, `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` will output the timestamp of `2023-10-10 10:10:10.123456789`.
+- The uppercase or lowercase of `MONTH`, `MON`, `DAY`, `DY` and formtas that output digits have same effect when used in `to_timestamp`, like `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month` can be replaced by `MONTH`, or `month`. The cases are ignored.
+- If multi times are specified for one component, the previous will be overwritten. Like `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, the output year will be `2022`.
+- To avoid unexpected time zone used during the convertion, it's recommended to put time zone in the ts string, e.g. '2023-10-10 10:10:10+08'. If time zone not specified, default will be that in server or client.
+- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with specified or default local timezone.
+- If `AM` or `PM` is specified in formats, the Hour must between `1-12`.
+- In some cases, `to_timestamp` can convert correctly even the format and the timestamp string are not totally matched. Like `to_timetamp('200101/2', 'yyyyMM1/dd')`, the digit `1` in format string are ignored, and the output timestsamp is `2001-01-02 00:00:00`. Spaces and tabs in formats and tiemstamp string are also ignored automatically.
+

 ### Time and Date Functions

--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@ -483,6 +483,91 @@ return_timestamp: {
 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。
 - return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。

+#### TO_CHAR
+
+```sql
+TO_CHAR(ts, format_str_literal)
+```
+
+**功能说明**: 将timestamp类型按照指定格式转换为字符串
+
+**返回结果数据类型**: VARCHAR
+
+**应用字段**: TIMESTAMP
+
+**嵌套子查询支持**: 适用于内层查询和外层查询
+
+**适用于**: 表和超级表
+
+**支持的格式**
+
+| **格式** | **说明**| **例子** |
+| --- | --- | --- |
+|AM,am,PM,pm| 无点分隔的上午下午 | 07:00:00am|
+|A.M.,a.m.,P.M.,p.m.| 有点分隔的上午下午| 07:00:00a.m.|
+|YYYY,yyyy|年, 4个及以上数字| 2023-10-10|
+|YYY,yyy| 年, 最后3位数字| 023-10-10|
+|YY,yy| 年, 最后2位数字| 23-10-10|
+|Y,y|年, 最后一位数字| 3-10-10|
+|MONTH|月, 全大写| 2023-JANUARY-01|
+|Month|月, 首字母大写| 2023-January-01|
+|month|月, 全小写| 2023-january-01|
+|MON| 月, 缩写, 全大写(三个字符)| JAN, SEP|
+|Mon| 月, 缩写, 首字母大写| Jan, Sep|
+|mon|月, 缩写, 全小写| jan, sep|
+|MM,mm|月, 数字 01-12|2023-01-01|
+|DD,dd|月日, 01-31||
+|DAY|周日, 全大写|MONDAY|
+|Day|周日, 首字符大写|Monday|
+|day|周日, 全小写|monday|
+|DY|周日, 缩写, 全大写|MON|
+|Dy|周日, 缩写, 首字符大写|Mon|
+|dy|周日, 缩写, 全小写|mon|
+|DDD|年日, 001-366||
+|D,d|周日, 数字, 1-7, Sunday(1) to Saturday(7)||
+|HH24,hh24|小时, 00-23|2023-01-30 23:59:59|
+|hh12,HH12, hh, HH| 小时, 01-12|2023-01-30 12:59:59PM|
+|MI,mi|分钟, 00-59||
+|SS,ss|秒, 00-59||
+|MS,ms|毫秒, 000-999||
+|US,us|微秒, 000000-999999||
+|NS,ns|纳秒, 000000000-999999999||
+|TZH,tzh|时区小时|2023-01-30 11:59:59PM +08|
+
+**使用说明**:
+- `Month`, `Day`等的输出格式是左对齐的, 右侧添加空格, 如`2023-OCTOBER  -01`, `2023-SEPTEMBER-01`, 9月是月份中英文字母数最长的, 因此9月没有空格. 星期类似.
+- 使用`ms`, `us`, `ns`时, 以上三种格式的输出只在精度上不同, 比如ts为 `1697182085123`,  `ms` 的输出为 `123`, `us` 的输出为 `123000`, `ns` 的输出为 `123000000`.
+- 时间格式中无法匹配规则的内容会直接输出. 如果想要在格式串中指定某些能够匹配规则的部分不做转换, 可以使用双引号, 如`to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. 如果想要输出双引号, 那么在双引号之前加一个反斜杠, 如 `to_char(ts, '\"yyyy-mm-dd\"')` 将会输出 `"2023-10-10"`.
+- 那些输出是数字的格式, 如`YYYY`, `DD`, 大写与小写意义相同, 即`yyyy` 和 `YYYY` 可以互换.
+- 推荐在时间格式中带时区信息，如果不带则默认输出的时区为服务端或客户端所配置的时区.
+
+#### TO_TIMESTAMP
+
+```sql
+TO_TIMESTAMP(ts_str_literal, format_str_literal)
+```
+
+**功能说明**: 将字符串按照指定格式转化为时间戳.
+
+**返回结果数据类型**: TIMESTAMP
+
+**应用字段**: VARCHAR
+
+**嵌套子查询支持**: 适用于内层查询和外层查询
+
+**适用于**: 表和超级表
+
+**支持的格式**: 与`to_char`相同
+
+**使用说明**:
+- 若`ms`, `us`, `ns`同时指定, 那么结果时间戳包含上述三个字段的和. 如 `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` 输出是 `2023-10-10 10:10:10.123456789`.
+- `MONTH`, `MON`, `DAY`, `DY` 以及其他输出为数字的格式的大小写意义相同, 如 `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month`可以被替换为`MONTH` 或者`Month`.
+- 如果同一字段被指定了多次, 那么前面的指定将会被覆盖.  如 `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, 输出年份是`2022`.
+- 为避免转换时使用了非预期的时区，推荐在时间中携带时区信息，例如'2023-10-10 10:10:10+08'，如果未指定时区则默认时区为服务端或客户端指定的时区。
+- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分.
+- 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12.
+- `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp('  23 年 - 1 月 - 01 日  ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换.
+

 ### 时间和日期函数

--- a/include/client/taos.h
+++ b/include/client/taos.h
@ -338,8 +338,11 @@ typedef struct tmq_raw_data {
 DLL_EXPORT int32_t     tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw);
 DLL_EXPORT int32_t     tmq_write_raw(TAOS *taos, tmq_raw_data raw);
 DLL_EXPORT int         taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname);
+DLL_EXPORT int         taos_write_raw_block_with_reqid(TAOS *taos, int numOfRows, char *pData, const char *tbname, int64_t reqid);
 DLL_EXPORT int         taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname,
                                                        TAOS_FIELD *fields, int numFields);
+DLL_EXPORT int         taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
+                                                        TAOS_FIELD *fields, int numFields, int64_t reqid);
 DLL_EXPORT void        tmq_free_raw(tmq_raw_data raw);

 // Returning null means error. Returned result need to be freed by tmq_free_json_meta
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@ -195,7 +195,8 @@ extern int64_t tsWalFsyncDataSizeLimit;
 // internal
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;
-extern int32_t tsStreamCheckpointTickInterval;
+extern int32_t tsStreamCheckpointInterval;
+extern float   tsSinkDataRate;
 extern int32_t tsStreamNodeCheckInterval;
 extern int32_t tsTtlUnit;
 extern int32_t tsTtlPushIntervalSec;
@ -204,9 +205,6 @@ extern int32_t tsTrimVDbIntervalSec;
 extern int32_t tsGrantHBInterval;
 extern int32_t tsUptimeInterval;

-extern int32_t tsRpcRetryLimit;
-extern int32_t tsRpcRetryInterval;
-
 extern bool    tsDisableStream;
 extern int64_t tsStreamBufferSize;
 extern bool    tsFilterScalarMode;
--- a/include/common/tgrant.h
+++ b/include/common/tgrant.h
@ -51,11 +51,7 @@ typedef enum {
 } EGrantType;

 int32_t grantCheck(EGrantType grant);
-#ifndef TD_GRANT_OPTIMIZE
-int32_t grantAlterActiveCode(const char* old, const char* newer, char* out, int8_t type);
-#else
 int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type);
-#endif

 #ifndef GRANTS_CFG
 #ifdef TD_ENTERPRISE
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@ -451,7 +451,7 @@ typedef struct SRetention {
  int8_t  keepUnit;
 } SRetention;

-#define RETENTION_VALID(r) (((r)->freq > 0) && ((r)->keep > 0))
+#define RETENTION_VALID(l, r) ((((l) == 0 && (r)->freq >= 0) || ((r)->freq > 0)) && ((r)->keep > 0))

 #pragma pack(push, 1)

@ -1568,6 +1568,7 @@ typedef struct {
 typedef struct {
  int32_t id;
  int8_t  isMnode;
+  int8_t  offlineReason;
  SEp     ep;
  char    active[TSDB_ACTIVE_KEY_LEN];
  char    connActive[TSDB_CONN_ACTIVE_KEY_LEN];
--- a/include/common/ttime.h
+++ b/include/common/ttime.h
@ -90,6 +90,34 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec

 void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);

+struct STm {
+  struct tm tm;
+  int64_t   fsec;  // in NANOSECOND
+};
+
+int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm);
+int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision);
+
+/// @brief convert a timestamp to a formatted string
+/// @param format the timestamp format, must null terminated
+/// @param [in,out] formats the formats array pointer generated. Shouldn't be NULL.
+/// If (*formats == NULL), [format] will be used and [formats] will be updated to the new generated
+/// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again.
+/// @param out output buffer, should be initialized by memset
+/// @notes remember to free the generated formats
+void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen);
+/// @brief convert a formatted timestamp string to a timestamp
+/// @param format must null terminated
+/// @param [in, out] formats, see taosTs2Char
+/// @param tsStr must null terminated
+/// @retval 0 for success, otherwise error occured
+/// @notes remember to free the generated formats even when error occured
+int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg,
+                    int32_t errMsgLen);
+
+void    TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen);
+int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr);
+
 #ifdef __cplusplus
 }
 #endif
--- a/include/common/ttokendef.h
+++ b/include/common/ttokendef.h
@ -246,120 +246,121 @@
 #define TK_INSERT                         227
 #define TK_NULL                           228
 #define TK_NK_QUESTION                    229
-#define TK_NK_ARROW                       230
-#define TK_ROWTS                          231
-#define TK_QSTART                         232
-#define TK_QEND                           233
-#define TK_QDURATION                      234
-#define TK_WSTART                         235
-#define TK_WEND                           236
-#define TK_WDURATION                      237
-#define TK_IROWTS                         238
-#define TK_ISFILLED                       239
-#define TK_CAST                           240
-#define TK_NOW                            241
-#define TK_TODAY                          242
-#define TK_TIMEZONE                       243
-#define TK_CLIENT_VERSION                 244
-#define TK_SERVER_VERSION                 245
-#define TK_SERVER_STATUS                  246
-#define TK_CURRENT_USER                   247
-#define TK_CASE                           248
-#define TK_WHEN                           249
-#define TK_THEN                           250
-#define TK_ELSE                           251
-#define TK_BETWEEN                        252
-#define TK_IS                             253
-#define TK_NK_LT                          254
-#define TK_NK_GT                          255
-#define TK_NK_LE                          256
-#define TK_NK_GE                          257
-#define TK_NK_NE                          258
-#define TK_MATCH                          259
-#define TK_NMATCH                         260
-#define TK_CONTAINS                       261
-#define TK_IN                             262
-#define TK_JOIN                           263
-#define TK_INNER                          264
-#define TK_SELECT                         265
-#define TK_NK_HINT                        266
-#define TK_DISTINCT                       267
-#define TK_WHERE                          268
-#define TK_PARTITION                      269
-#define TK_BY                             270
-#define TK_SESSION                        271
-#define TK_STATE_WINDOW                   272
-#define TK_EVENT_WINDOW                   273
-#define TK_SLIDING                        274
-#define TK_FILL                           275
-#define TK_VALUE                          276
-#define TK_VALUE_F                        277
-#define TK_NONE                           278
-#define TK_PREV                           279
-#define TK_NULL_F                         280
-#define TK_LINEAR                         281
-#define TK_NEXT                           282
-#define TK_HAVING                         283
-#define TK_RANGE                          284
-#define TK_EVERY                          285
-#define TK_ORDER                          286
-#define TK_SLIMIT                         287
-#define TK_SOFFSET                        288
-#define TK_LIMIT                          289
-#define TK_OFFSET                         290
-#define TK_ASC                            291
-#define TK_NULLS                          292
-#define TK_ABORT                          293
-#define TK_AFTER                          294
-#define TK_ATTACH                         295
-#define TK_BEFORE                         296
-#define TK_BEGIN                          297
-#define TK_BITAND                         298
-#define TK_BITNOT                         299
-#define TK_BITOR                          300
-#define TK_BLOCKS                         301
-#define TK_CHANGE                         302
-#define TK_COMMA                          303
-#define TK_CONCAT                         304
-#define TK_CONFLICT                       305
-#define TK_COPY                           306
-#define TK_DEFERRED                       307
-#define TK_DELIMITERS                     308
-#define TK_DETACH                         309
-#define TK_DIVIDE                         310
-#define TK_DOT                            311
-#define TK_EACH                           312
-#define TK_FAIL                           313
-#define TK_FILE                           314
-#define TK_FOR                            315
-#define TK_GLOB                           316
-#define TK_ID                             317
-#define TK_IMMEDIATE                      318
-#define TK_IMPORT                         319
-#define TK_INITIALLY                      320
-#define TK_INSTEAD                        321
-#define TK_ISNULL                         322
-#define TK_KEY                            323
-#define TK_MODULES                        324
-#define TK_NK_BITNOT                      325
-#define TK_NK_SEMI                        326
-#define TK_NOTNULL                        327
-#define TK_OF                             328
-#define TK_PLUS                           329
-#define TK_PRIVILEGE                      330
-#define TK_RAISE                          331
-#define TK_RESTRICT                       332
-#define TK_ROW                            333
-#define TK_SEMI                           334
-#define TK_STAR                           335
-#define TK_STATEMENT                      336
-#define TK_STRICT                         337
-#define TK_STRING                         338
-#define TK_TIMES                          339
-#define TK_VALUES                         340
-#define TK_VARIABLE                       341
-#define TK_VIEW                           342
-#define TK_WAL                            343
+#define TK_NK_ALIAS                       230
+#define TK_NK_ARROW                       231
+#define TK_ROWTS                          232
+#define TK_QSTART                         233
+#define TK_QEND                           234
+#define TK_QDURATION                      235
+#define TK_WSTART                         236
+#define TK_WEND                           237
+#define TK_WDURATION                      238
+#define TK_IROWTS                         239
+#define TK_ISFILLED                       240
+#define TK_CAST                           241
+#define TK_NOW                            242
+#define TK_TODAY                          243
+#define TK_TIMEZONE                       244
+#define TK_CLIENT_VERSION                 245
+#define TK_SERVER_VERSION                 246
+#define TK_SERVER_STATUS                  247
+#define TK_CURRENT_USER                   248
+#define TK_CASE                           249
+#define TK_WHEN                           250
+#define TK_THEN                           251
+#define TK_ELSE                           252
+#define TK_BETWEEN                        253
+#define TK_IS                             254
+#define TK_NK_LT                          255
+#define TK_NK_GT                          256
+#define TK_NK_LE                          257
+#define TK_NK_GE                          258
+#define TK_NK_NE                          259
+#define TK_MATCH                          260
+#define TK_NMATCH                         261
+#define TK_CONTAINS                       262
+#define TK_IN                             263
+#define TK_JOIN                           264
+#define TK_INNER                          265
+#define TK_SELECT                         266
+#define TK_NK_HINT                        267
+#define TK_DISTINCT                       268
+#define TK_WHERE                          269
+#define TK_PARTITION                      270
+#define TK_BY                             271
+#define TK_SESSION                        272
+#define TK_STATE_WINDOW                   273
+#define TK_EVENT_WINDOW                   274
+#define TK_SLIDING                        275
+#define TK_FILL                           276
+#define TK_VALUE                          277
+#define TK_VALUE_F                        278
+#define TK_NONE                           279
+#define TK_PREV                           280
+#define TK_NULL_F                         281
+#define TK_LINEAR                         282
+#define TK_NEXT                           283
+#define TK_HAVING                         284
+#define TK_RANGE                          285
+#define TK_EVERY                          286
+#define TK_ORDER                          287
+#define TK_SLIMIT                         288
+#define TK_SOFFSET                        289
+#define TK_LIMIT                          290
+#define TK_OFFSET                         291
+#define TK_ASC                            292
+#define TK_NULLS                          293
+#define TK_ABORT                          294
+#define TK_AFTER                          295
+#define TK_ATTACH                         296
+#define TK_BEFORE                         297
+#define TK_BEGIN                          298
+#define TK_BITAND                         299
+#define TK_BITNOT                         300
+#define TK_BITOR                          301
+#define TK_BLOCKS                         302
+#define TK_CHANGE                         303
+#define TK_COMMA                          304
+#define TK_CONCAT                         305
+#define TK_CONFLICT                       306
+#define TK_COPY                           307
+#define TK_DEFERRED                       308
+#define TK_DELIMITERS                     309
+#define TK_DETACH                         310
+#define TK_DIVIDE                         311
+#define TK_DOT                            312
+#define TK_EACH                           313
+#define TK_FAIL                           314
+#define TK_FILE                           315
+#define TK_FOR                            316
+#define TK_GLOB                           317
+#define TK_ID                             318
+#define TK_IMMEDIATE                      319
+#define TK_IMPORT                         320
+#define TK_INITIALLY                      321
+#define TK_INSTEAD                        322
+#define TK_ISNULL                         323
+#define TK_KEY                            324
+#define TK_MODULES                        325
+#define TK_NK_BITNOT                      326
+#define TK_NK_SEMI                        327
+#define TK_NOTNULL                        328
+#define TK_OF                             329
+#define TK_PLUS                           330
+#define TK_PRIVILEGE                      331
+#define TK_RAISE                          332
+#define TK_RESTRICT                       333
+#define TK_ROW                            334
+#define TK_SEMI                           335
+#define TK_STAR                           336
+#define TK_STATEMENT                      337
+#define TK_STRICT                         338
+#define TK_STRING                         339
+#define TK_TIMES                          340
+#define TK_VALUES                         341
+#define TK_VARIABLE                       342
+#define TK_VIEW                           343
+#define TK_WAL                            344



--- a/include/libs/function/functionMgt.h
+++ b/include/libs/function/functionMgt.h
@ -94,6 +94,8 @@ typedef enum EFunctionType {
  FUNCTION_TYPE_TO_ISO8601,
  FUNCTION_TYPE_TO_UNIXTIMESTAMP,
  FUNCTION_TYPE_TO_JSON,
+  FUNCTION_TYPE_TO_TIMESTAMP,
+  FUNCTION_TYPE_TO_CHAR,

  // date and time function
  FUNCTION_TYPE_NOW = 2500,
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@ -80,6 +80,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toTimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toCharFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@ -38,16 +38,25 @@ extern "C" {
 #define TASK_DOWNSTREAM_READY       0x0
 #define TASK_DOWNSTREAM_NOT_READY   0x1
 #define TASK_DOWNSTREAM_NOT_LEADER  0x2
-#define TASK_SELF_NEW_STAGE         0x3
+#define TASK_UPSTREAM_NEW_STAGE     0x3

 #define NODE_ROLE_UNINIT     0x1
 #define NODE_ROLE_LEADER     0x2
 #define NODE_ROLE_FOLLOWER   0x3

+#define HAS_RELATED_FILLHISTORY_TASK(_t)  ((_t)->hTaskInfo.id.taskId != 0)
+#define CLEAR_RELATED_FILLHISTORY_TASK(_t) \
+  do {                                     \
+    (_t)->hTaskInfo.id.taskId = 0;         \
+    (_t)->hTaskInfo.id.streamId = 0;       \
+  } while (0)
+
 typedef struct SStreamTask   SStreamTask;
 typedef struct SStreamQueue  SStreamQueue;
+typedef struct SStreamTaskSM SStreamTaskSM;

 #define SSTREAM_TASK_VER 2
+
 enum {
  STREAM_STATUS__NORMAL = 0,
  STREAM_STATUS__STOP,
@ -58,7 +67,7 @@ enum {
 };

 typedef enum ETaskStatus {
-  TASK_STATUS__NORMAL = 0,
+  TASK_STATUS__READY = 0,
  TASK_STATUS__DROPPING,
  TASK_STATUS__UNINIT,  // not used, an placeholder
  TASK_STATUS__STOP,
@ -66,6 +75,7 @@ typedef enum ETaskStatus {
  TASK_STATUS__HALT,          // pause, but not be manipulated by user command
  TASK_STATUS__PAUSE,         // pause
  TASK_STATUS__CK,            // stream task is in checkpoint status, no data are allowed to put into inputQ anymore
+  TASK_STATUS__STREAM_SCAN_HISTORY,
 } ETaskStatus;

 enum {
@ -118,6 +128,22 @@ enum {
  STREAM_META_OK_TO_STOP = 2,
 };

+typedef enum EStreamTaskEvent {
+  TASK_EVENT_INIT = 0x1,
+  TASK_EVENT_INIT_SCANHIST = 0x2,
+  TASK_EVENT_INIT_STREAM_SCANHIST = 0x3,
+  TASK_EVENT_SCANHIST_DONE = 0x4,
+  TASK_EVENT_STOP = 0x5,
+  TASK_EVENT_GEN_CHECKPOINT = 0x6,
+  TASK_EVENT_CHECKPOINT_DONE = 0x7,
+  TASK_EVENT_PAUSE = 0x8,
+  TASK_EVENT_RESUME = 0x9,
+  TASK_EVENT_HALT = 0xA,
+  TASK_EVENT_DROPPING = 0xB,
+  TASK_EVENT_SCAN_TSDB = 0xC,
+  TASK_EVENT_SCAN_WAL = 0xD,
+} EStreamTaskEvent;
+
 typedef struct {
  int8_t type;
 } SStreamQueueItem;
@ -155,11 +181,6 @@ typedef struct {
  SSDataBlock* pBlock;
 } SStreamRefDataBlock;

-typedef struct {
-  int8_t       type;
-  SSDataBlock* pBlock;
-} SStreamTrigger;
-
 typedef struct SStreamQueueNode SStreamQueueNode;

 struct SStreamQueueNode {
@ -215,6 +236,11 @@ typedef struct {
  SUseDbRsp dbInfo;
 } STaskDispatcherShuffle;

+typedef struct {
+  int32_t nodeId;
+  SEpSet  epset;
+} SDownstreamTaskEpset;
+
 typedef struct {
  int64_t         stbUid;
  char            stbFullName[TSDB_TABLE_FNAME_LEN];
@ -265,6 +291,7 @@ typedef struct SCheckpointInfo {
 } SCheckpointInfo;

 typedef struct SStreamStatus {
+  SStreamTaskSM* pSM;
  int8_t         taskStatus;
  int8_t         downstreamReady;  // downstream tasks are all ready now, if this flag is set
  int8_t         schedStatus;
@ -305,15 +332,10 @@ typedef struct SDispatchMsgInfo {
  void*   pTimer; // used to dispatch data after a given time duration
 } SDispatchMsgInfo;

-typedef struct STaskOutputQueue {
+typedef struct STaskQueue {
  int8_t        status;
  SStreamQueue* queue;
-} STaskOutputQueue;
-
-typedef struct STaskInputInfo {
-  int8_t        status;
-  SStreamQueue* queue;
-} STaskInputInfo;
+} STaskQueue;

 typedef struct STaskSchedInfo {
  int8_t  status;
@ -349,6 +371,7 @@ typedef struct SHistoryTaskInfo {
  int32_t tickCount;
  int32_t retryTimes;
  int32_t waitInterval;
+  int64_t haltVer;      // offset in wal when halt the stream task
 } SHistoryTaskInfo;

 typedef struct STaskOutputInfo {
@ -361,6 +384,7 @@ typedef struct STaskOutputInfo {
  };
  int8_t        type;
  STokenBucket* pTokenBucket;
+  SArray*       pDownstreamUpdateList;
 } STaskOutputInfo;

 typedef struct SUpstreamInfo {
@ -372,8 +396,8 @@ struct SStreamTask {
  int64_t             ver;
  SStreamTaskId       id;
  SSTaskBasicInfo     info;
-  STaskOutputQueue    outputq;
-  STaskInputInfo      inputInfo;
+  STaskQueue          outputq;
+  STaskQueue          inputq;
  STaskSchedInfo      schedInfo;
  STaskOutputInfo     outputInfo;
  SDispatchMsgInfo    msgInfo;
@ -408,11 +432,16 @@ struct SStreamTask {
 typedef struct STaskStartInfo {
  int64_t   startTs;
  int64_t   readyTs;
-  int32_t   startedAfterNodeUpdate;
+  int32_t   startAllTasksFlag;
  SHashObj* pReadyTaskSet;           // tasks that are all ready for running stream processing
  int32_t   elapsedTime;
 } STaskStartInfo;

+typedef struct STaskUpdateInfo {
+  SHashObj* pTasks;
+  int32_t   transId;
+} STaskUpdateInfo;
+
 // meta
 typedef struct SStreamMeta {
  char*           path;
@ -435,7 +464,7 @@ typedef struct SStreamMeta {
  SHashObj*       pTaskBackendUnique;
  TdThreadMutex   backendMutex;
  SMetaHbInfo*    pHbInfo;
-  SHashObj*     pUpdateTaskSet;
+  STaskUpdateInfo updateInfo;
  int32_t         numOfStreamTasks;  // this value should be increased when a new task is added into the meta
  int32_t         numOfPausedTasks;
  int32_t         chkptNotReadyTasks;
@ -617,7 +646,8 @@ typedef struct STaskStatusEntry {
 typedef struct SStreamHbMsg {
  int32_t vgId;
  int32_t numOfTasks;
-  SArray* pTaskStatus;  // SArray<SStreamTaskStatusEntry>
+  SArray* pTaskStatus;  // SArray<STaskStatusEntry>
+  SArray* pUpdateNodes; // SArray<int32_t>, needs update the epsets in stream tasks for those nodes.
 } SStreamHbMsg;

 int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp);
@ -641,6 +671,7 @@ typedef struct SNodeUpdateInfo {
 } SNodeUpdateInfo;

 typedef struct SStreamTaskNodeUpdateMsg {
+  int32_t transId;   // to identify the msg
  int64_t streamId;
  int32_t taskId;
  SArray* pNodeList;  // SArray<SNodeUpdateInfo>
@ -686,24 +717,36 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t
 void    streamTaskInputFail(SStreamTask* pTask);
 int32_t streamExecTask(SStreamTask* pTask);
 int32_t streamSchedExec(SStreamTask* pTask);
-bool    streamTaskShouldStop(const SStreamStatus* pStatus);
-bool    streamTaskShouldPause(const SStreamStatus* pStatus);
+bool    streamTaskShouldStop(const SStreamTask* pStatus);
+bool    streamTaskShouldPause(const SStreamTask* pStatus);
 bool    streamTaskIsIdle(const SStreamTask* pTask);
+bool    streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
+
+char*       createStreamTaskIdStr(int64_t streamId, int32_t taskId);
+ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr);
+const char* streamTaskGetStatusStr(ETaskStatus status);
+void        streamTaskResetStatus(SStreamTask* pTask);
+void        streamTaskSetStatusReady(SStreamTask* pTask);

 void    initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);

-char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
-
 // recover and fill history
 void    streamTaskCheckDownstream(SStreamTask* pTask);
-int32_t streamTaskStartScanHistory(SStreamTask* pTask);
+int32_t onNormalTaskReady(SStreamTask* pTask);
+int32_t onScanhistoryTaskReady(SStreamTask* pTask);
+
 int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage);
 int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
 void    streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
 bool    streamTaskAllUpstreamClosed(SStreamTask* pTask);
 bool    streamTaskSetSchedStatusWait(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusActive(SStreamTask* pTask);
-int8_t  streamTaskSetSchedStatusInActive(SStreamTask* pTask);
+int8_t  streamTaskSetSchedStatusInactive(SStreamTask* pTask);
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask);
+
+int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
+void    streamTaskRestoreStatus(SStreamTask* pTask);

 int32_t streamTaskStop(SStreamTask* pTask);
 int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp,
@ -714,16 +757,11 @@ int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask);
 int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
 bool    streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
-int32_t streamQueueGetAvailableSpace(const SStreamQueue* pQueue, int32_t* availNum, double* availSize);

 // common
 int32_t     streamRestoreParam(SStreamTask* pTask);
-int32_t     streamSetStatusNormal(SStreamTask* pTask);
-int32_t     streamSetStatusUnint(SStreamTask* pTask);
-const char* streamGetTaskStatusStr(int32_t status);
 void        streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta);
-void        streamTaskResume(SStreamTask* pTask, SStreamMeta* pMeta);
-void        streamTaskResumeFromHalt(SStreamTask* pTask);
+void        streamTaskResume(SStreamTask* pTask);
 void        streamTaskDisablePause(SStreamTask* pTask);
 void        streamTaskEnablePause(SStreamTask* pTask);
 int32_t     streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask);
@ -759,7 +797,6 @@ int32_t      streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
 int32_t      streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
 int32_t      streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);
-int32_t      streamMetaGetNumOfStreamTasks(SStreamMeta* pMeta);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaReopen(SStreamMeta* pMeta);
@ -768,12 +805,13 @@ int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
 void         streamMetaStartHb(SStreamMeta* pMeta);
 void         streamMetaInitForSnode(SStreamMeta* pMeta);
+bool         streamMetaTaskInTimer(SStreamMeta* pMeta);
+int32_t      streamMetaUpdateTaskReadyInfo(SStreamTask* pTask);

 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask);
-
 int32_t streamAlignTransferState(SStreamTask* pTask);
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask,
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@ -46,6 +46,7 @@ extern "C" {
 #define SYNC_HEARTBEAT_SLOW_MS       1500
 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500
 #define SYNC_SNAP_RESEND_MS          1000 * 60
+#define SYNC_SNAP_TIMEOUT_MS         1000 * 600

 #define SYNC_VND_COMMIT_MIN_MS 3000

--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@ -332,6 +332,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_INVALID_REPLICA           TAOS_DEF_ERROR_CODE(0, 0x03B7)
 #define TSDB_CODE_MND_DNODE_IN_CREATING         TAOS_DEF_ERROR_CODE(0, 0x03B8)
 #define TSDB_CODE_MND_DNODE_IN_DROPPING         TAOS_DEF_ERROR_CODE(0, 0x03B9)
+#define TSDB_CODE_MND_NO_ENOUGH_VNODES          TAOS_DEF_ERROR_CODE(0, 0x03BA)

 // mnode-stable-part2
 #define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC  TAOS_DEF_ERROR_CODE(0, 0x03C0)
@ -657,6 +658,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 #define TSDB_CODE_PAR_INVALID_DB_OPTION         TAOS_DEF_ERROR_CODE(0, 0x2619)
 #define TSDB_CODE_PAR_INVALID_TABLE_OPTION      TAOS_DEF_ERROR_CODE(0, 0x261A)
+#define TSDB_CODE_PAR_INTER_VALUE_TOO_BIG       TAOS_DEF_ERROR_CODE(0, 0x261B)
 #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST    TAOS_DEF_ERROR_CODE(0, 0x2624)
 #define TSDB_CODE_PAR_AGG_FUNC_NESTING          TAOS_DEF_ERROR_CODE(0, 0x2627)
 #define TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE    TAOS_DEF_ERROR_CODE(0, 0x2628)
@ -739,6 +741,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE       TAOS_DEF_ERROR_CODE(0, 0x2803)
 #define TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION      TAOS_DEF_ERROR_CODE(0, 0x2804)
 #define TSDB_CODE_FUNC_DUP_TIMESTAMP            TAOS_DEF_ERROR_CODE(0, 0x2805)
+#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED      TAOS_DEF_ERROR_CODE(0, 0x2806)

 //udf
 #define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)
--- a/include/util/tconfig.h
+++ b/include/util/tconfig.h
@ -65,7 +65,6 @@ typedef struct SConfigItem {
  union {
    bool    bval;
    float   fval;
-    double  dval;
    int32_t i32;
    int64_t i64;
    char   *str;
@ -104,7 +103,6 @@ int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scop
 int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
 int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
 int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope);
-int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope);
 int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
 int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
 int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@ -109,6 +109,15 @@ extern const int32_t TYPE_BYTES[21];

 #define TSDB_INS_USER_STABLES_DBNAME_COLID 2

+static const int64_t TICK_PER_SECOND[] = {
+  1000LL, // MILLISECOND
+  1000000LL, // MICROSECOND
+  1000000000LL, // NANOSECOND
+  0LL, // HOUR
+  0LL, // MINUTE
+  1LL // SECOND
+};
+
 #define TSDB_TICK_PER_SECOND(precision)               \
  ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI \
                 ? 1000LL                             \
@ -294,6 +303,8 @@ typedef enum ELogicConditionType {
 #define TSDB_SYNC_APPLYQ_SIZE_LIMIT    512
 #define TSDB_SYNC_NEGOTIATION_WIN      512

+#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048
+
 #define TSDB_TBNAME_COLUMN_INDEX     (-1)
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta

@ -316,7 +327,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_DAYS_PER_FILE          (3650 * 1440)
 #define TSDB_DEFAULT_DAYS_PER_FILE      (10 * 1440)
 #define TSDB_MIN_DURATION_PER_FILE      60  // unit minute
-#define TSDB_MAX_DURATION_PER_FILE      (3650 * 1440)
+#define TSDB_MAX_DURATION_PER_FILE      (90 * 1440)
 #define TSDB_DEFAULT_DURATION_PER_FILE  (10 * 1440)
 #define TSDB_MIN_KEEP                   (1 * 1440)          // data in db to be reserved. unit minute
 #define TSDB_MAX_KEEP                   (365000 * 1440)     // data in db to be reserved.
--- a/include/util/tqueue.h
+++ b/include/util/tqueue.h
@ -101,6 +101,9 @@ struct STaosQall {
  STaosQnode *current;
  STaosQnode *start;
  int32_t     numOfItems;
+  int64_t     memOfItems;
+  int32_t     unAccessedNumOfItems;
+  int64_t     unAccessMemOfItems;
 };

 STaosQueue *taosOpenQueue();
@ -123,6 +126,9 @@ int32_t    taosReadAllQitems(STaosQueue *queue, STaosQall *qall);
 int32_t    taosGetQitem(STaosQall *qall, void **ppItem);
 void       taosResetQitems(STaosQall *qall);
 int32_t    taosQallItemSize(STaosQall *qall);
+int64_t    taosQallMemSize(STaosQall *qll);
+int64_t    taosQallUnAccessedItemSize(STaosQall *qall);
+int64_t    taosQallUnAccessedMemSize(STaosQall *qall);

 STaosQset *taosOpenQset();
 void       taosCloseQset(STaosQset *qset);
@ -135,8 +141,6 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo)
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);

-extern int64_t tsRpcQueueMemoryAllowed;
-
 #ifdef __cplusplus
 }
 #endif
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@ -1338,6 +1338,11 @@ end:

 int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields,
                                     int numFields) {
+  return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0);
+}
+
+int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
+    TAOS_FIELD *fields, int numFields, int64_t reqid){
  if (!taos || !pData || !tbname) {
    return TSDB_CODE_INVALID_PARA;
  }
@ -1347,7 +1352,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
  SHashObj*   pVgHash = NULL;
  uDebug("taos_write_raw_block_with_fields called");

-  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
+  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
  if (!pRequest) {
    uError("WriteRaw:createRequest error request is null");
    code = terrno;
@ -1427,6 +1432,10 @@ end:
 }

 int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) {
+  return taos_write_raw_block_with_reqid(taos, rows, pData, tbname, 0);
+}
+
+int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, int64_t reqid) {
  if (!taos || !pData || !tbname) {
    return TSDB_CODE_INVALID_PARA;
  }
@ -1436,7 +1445,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
  SHashObj*   pVgHash = NULL;

  uDebug("taos_write_raw_block called");
-  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
+  SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
  if (!pRequest) {
    uError("WriteRaw:createRequest error request is null");
    code = terrno;
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@ -104,13 +104,21 @@ uint16_t tsAuditPort = 6043;
 bool     tsEnableAuditCreateTable = true;

 // telem
+#ifdef TD_ENTERPRISE
+bool     tsEnableTelem = false;
+#else
 bool     tsEnableTelem = true;
+#endif
 int32_t  tsTelemInterval = 43200;
 char     tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com";
 uint16_t tsTelemPort = 80;
 char    *tsTelemUri = "/report";

+#ifdef TD_ENTERPRISE
+bool  tsEnableCrashReport = false;
+#else
 bool  tsEnableCrashReport = true;
+#endif
 char *tsClientCrashReportUri = "/ccrashreport";
 char *tsSvrCrashReportUri = "/dcrashreport";

@ -246,7 +254,8 @@ int32_t tsTtlBatchDropNum = 10000;   // number of tables dropped per batch
 // internal
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
-int32_t tsStreamCheckpointTickInterval = 300;
+int32_t tsStreamCheckpointInterval = 60;
+float   tsSinkDataRate = 2.0;
 int32_t tsStreamNodeCheckInterval = 30;
 int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushIntervalSec = 10;
@ -275,8 +284,6 @@ int32_t tsS3BlockSize = 4096;     // number of tsdb pages
 int32_t tsS3BlockCacheSize = 16;  // number of blocks
 int32_t tsS3UploadDelaySec = 60 * 60;

-int32_t tsCheckpointInterval = 300;
-
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
  SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -506,11 +513,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
  if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1;
  if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1;

-  if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "AVX", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "FMA", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1;
  if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1;

  if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1;
@ -667,19 +674,18 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {

  if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1;
  if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointTickInterval, 60, 1200, CFG_SCOPE_SERVER) != 0)
-    return -1;
+  if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER) != 0) return -1;

-  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0)
-    return -1;
+  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1;

-  if (cfgAddString(pCfg, "LossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddFloat(pCfg, "FPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddDouble(pCfg, "DPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "MaxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "CurRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddBool(pCfg, "IfAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddString(pCfg, "Compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;

  if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
  if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
@ -1044,7 +1050,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
  tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
  tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;

-  tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "SIMD-builtins")->bval;
+  tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "simdEnable")->bval;
  tsTagFilterCache = (bool)cfgGetItem(pCfg, "tagFilterCache")->bval;

  tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
@ -1103,16 +1109,18 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {

  tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;

-  tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "LossyColumns")->str, sizeof(tsLossyColumns));
-  tsFPrecision = cfgGetItem(pCfg, "FPrecision")->fval;
-  tsDPrecision = cfgGetItem(pCfg, "DPrecision")->dval;
-  tsMaxRange = cfgGetItem(pCfg, "MaxRange")->i32;
-  tsCurRange = cfgGetItem(pCfg, "CurRange")->i32;
-  tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
-  tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
+  tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "lossyColumns")->str, sizeof(tsLossyColumns));
+  tsFPrecision = cfgGetItem(pCfg, "fPrecision")->fval;
+  tsDPrecision = cfgGetItem(pCfg, "dPrecision")->fval;
+  tsMaxRange = cfgGetItem(pCfg, "maxRange")->i32;
+  tsCurRange = cfgGetItem(pCfg, "curRange")->i32;
+  tsIfAdtFse = cfgGetItem(pCfg, "ifAdtFse")->bval;
+  tstrncpy(tsCompressor, cfgGetItem(pCfg, "compressor")->str, sizeof(tsCompressor));

  tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
  tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
+  tsStreamCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i32;
+  tsSinkDataRate = cfgGetItem(pCfg, "streamSinkDataRate")->fval;

  tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval;
  tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32;
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@ -8405,7 +8405,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
      } else {
        tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE);
      }
-      taosMemoryFree(pTbData->pCreateTbReq);
+      taosMemoryFreeClear(pTbData->pCreateTbReq);
    }

    if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
--- a/source/common/src/ttime.c
+++ b/source/common/src/ttime.c
@ -25,7 +25,6 @@

 #include "tlog.h"

-
 // ==== mktime() kernel code =================//
 static int64_t m_deltaUtc = 0;

@ -682,7 +681,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
  }

  // The following code handles the y/n time duration
-  int64_t numOfMonth = (unit == 'y')? duration*12:duration;
+  int64_t numOfMonth = (unit == 'y') ? duration * 12 : duration;
  int64_t fraction = t % TSDB_TICK_PER_SECOND(precision);

  struct tm tm;
@ -768,7 +767,6 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
  int32_t precision = pInterval->precision;

  if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) {
-
    start /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
    struct tm tm;
    time_t    tt = (time_t)start;
@ -799,7 +797,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
        int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1;

        if (newe < ts) {  // move towards the greater endpoint
-          while(newe < ts && news < ts) {
+          while (newe < ts && news < ts) {
            news += pInterval->sliding;
            newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1;
          }
@ -978,3 +976,947 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)

  tstrncpy(buf, ts, bufLen);
 }
+
+int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) {
+  tm->fsec = ts % TICK_PER_SECOND[precision] * (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]);
+  time_t t = ts / TICK_PER_SECOND[precision];
+  taosLocalTime(&t, &tm->tm, NULL);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision) {
+  *ts = taosMktime(&tm->tm);
+  *ts *= TICK_PER_SECOND[precision];
+  *ts += tm->fsec / (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]);
+  return TSDB_CODE_SUCCESS;
+}
+
+typedef struct {
+  const char* name;
+  int         len;
+  int         id;
+  bool        isDigit;
+} TSFormatKeyWord;
+
+typedef enum {
+  // TSFKW_AD,   // BC AD
+  // TSFKW_A_D,  // A.D. B.C.
+  TSFKW_AM,   // AM, PM
+  TSFKW_A_M,  // A.M., P.M.
+  // TSFKW_BC,   // BC AD
+  // TSFKW_B_C,  // B.C. A.D.
+  TSFKW_DAY,  // MONDAY, TUESDAY ...
+  TSFKW_DDD,  // Day of year 001-366
+  TSFKW_DD,   // Day of month 01-31
+  TSFKW_Day,  // Sunday, Monday
+  TSFKW_DY,   // MON, TUE
+  TSFKW_Dy,   // Mon, Tue
+  TSFKW_D,    // 1-7 -> Sunday(1) -> Saturday(7)
+  TSFKW_HH24,
+  TSFKW_HH12,
+  TSFKW_HH,
+  TSFKW_MI,  // minute
+  TSFKW_MM,
+  TSFKW_MONTH,  // JANUARY, FEBRUARY
+  TSFKW_MON,
+  TSFKW_Month,
+  TSFKW_Mon,
+  TSFKW_MS,
+  TSFKW_NS,
+  //TSFKW_OF,
+  TSFKW_PM,
+  TSFKW_P_M,
+  TSFKW_SS,
+  TSFKW_TZH,
+  // TSFKW_TZM,
+  // TSFKW_TZ,
+  TSFKW_US,
+  TSFKW_YYYY,
+  TSFKW_YYY,
+  TSFKW_YY,
+  TSFKW_Y,
+  // TSFKW_a_d,
+  // TSFKW_ad,
+  TSFKW_am,
+  TSFKW_a_m,
+  // TSFKW_b_c,
+  // TSFKW_bc,
+  TSFKW_day,
+  TSFKW_ddd,
+  TSFKW_dd,
+  TSFKW_dy,   // mon, tue
+  TSFKW_d,
+  TSFKW_hh24,
+  TSFKW_hh12,
+  TSFKW_hh,
+  TSFKW_mi,
+  TSFKW_mm,
+  TSFKW_month,
+  TSFKW_mon,
+  TSFKW_ms,
+  TSFKW_ns,
+  TSFKW_pm,
+  TSFKW_p_m,
+  TSFKW_ss,
+  TSFKW_tzh,
+  // TSFKW_tzm,
+  // TSFKW_tz,
+  TSFKW_us,
+  TSFKW_yyyy,
+  TSFKW_yyy,
+  TSFKW_yy,
+  TSFKW_y,
+  TSFKW_last_
+} TSFormatKeywordId;
+
+// clang-format off
+static const TSFormatKeyWord formatKeyWords[] = {
+  //{"AD", 2, TSFKW_AD, false},
+  //{"A.D.", 4, TSFKW_A_D},
+  {"AM", 2, TSFKW_AM, false},
+  {"A.M.", 4, TSFKW_A_M, false},
+  //{"BC", 2, TSFKW_BC, false},
+  //{"B.C.", 4, TSFKW_B_C, false},
+  {"DAY", 3, TSFKW_DAY, false},
+  {"DDD", 3, TSFKW_DDD, true},
+  {"DD", 2, TSFKW_DD, true},
+  {"Day", 3, TSFKW_Day, false},
+  {"DY", 2, TSFKW_DY, false},
+  {"Dy", 2, TSFKW_Dy, false},
+  {"D", 1, TSFKW_D, true},
+  {"HH24", 4, TSFKW_HH24, true},
+  {"HH12", 4, TSFKW_HH12, true},
+  {"HH", 2, TSFKW_HH, true},
+  {"MI", 2, TSFKW_MI, true},
+  {"MM", 2, TSFKW_MM, true},
+  {"MONTH", 5, TSFKW_MONTH, false},
+  {"MON", 3, TSFKW_MON, false},
+  {"Month", 5, TSFKW_Month, false},
+  {"Mon", 3, TSFKW_Mon, false},
+  {"MS", 2, TSFKW_MS, true},
+  {"NS", 2, TSFKW_NS, true},
+  //{"OF", 2, TSFKW_OF, false},
+  {"PM", 2, TSFKW_PM, false},
+  {"P.M.", 4, TSFKW_P_M, false},
+  {"SS", 2, TSFKW_SS, true},
+  {"TZH", 3, TSFKW_TZH, false},
+  //{"TZM", 3, TSFKW_TZM},
+  //{"TZ", 2, TSFKW_TZ},
+  {"US", 2, TSFKW_US, true},
+  {"YYYY", 4, TSFKW_YYYY, true},
+  {"YYY", 3, TSFKW_YYY, true},
+  {"YY", 2, TSFKW_YY, true},
+  {"Y", 1, TSFKW_Y, true},
+  //{"a.d.", 4, TSFKW_a_d, false},
+  //{"ad", 2, TSFKW_ad, false},
+  {"am", 2, TSFKW_am, false},
+  {"a.m.", 4, TSFKW_a_m, false},
+  //{"b.c.", 4, TSFKW_b_c, false},
+  //{"bc", 2, TSFKW_bc, false},
+  {"day", 3, TSFKW_day, false},
+  {"ddd", 3, TSFKW_DDD, true},
+  {"dd", 2, TSFKW_DD, true},
+  {"dy", 2, TSFKW_dy, false},
+  {"d", 1, TSFKW_D, true},
+  {"hh24", 4, TSFKW_HH24, true},
+  {"hh12", 4, TSFKW_HH12, true},
+  {"hh", 2, TSFKW_HH, true},
+  {"mi", 2, TSFKW_MI, true},
+  {"mm", 2, TSFKW_MM, true},
+  {"month", 5, TSFKW_month, false},
+  {"mon", 3, TSFKW_mon, false},
+  {"ms", 2, TSFKW_MS, true},
+  {"ns", 2, TSFKW_NS, true},
+  //{"of", 2, TSFKW_OF, false},
+  {"pm", 2, TSFKW_pm, false},
+  {"p.m.", 4, TSFKW_p_m, false},
+  {"ss", 2, TSFKW_SS, true},
+  {"tzh", 3, TSFKW_TZH, false},
+  //{"tzm", 3, TSFKW_TZM},
+  //{"tz", 2, TSFKW_tz},
+  {"us", 2, TSFKW_US, true},
+  {"yyyy", 4, TSFKW_YYYY, true},
+  {"yyy", 3, TSFKW_YYY, true},
+  {"yy", 2, TSFKW_YY, true},
+  {"y", 1, TSFKW_Y, true},
+  {NULL, 0, 0}
+};
+// clang-format on
+
+#define TS_FROMAT_KEYWORD_INDEX_SIZE ('z' - 'A' + 1)
+static const int TSFormatKeywordIndex[TS_FROMAT_KEYWORD_INDEX_SIZE] = {
+    /*A*/ TSFKW_AM,     -1, -1,
+    /*D*/ TSFKW_DAY,    -1, -1, -1,
+    /*H*/ TSFKW_HH24,   -1, -1, -1, -1,
+    /*M*/ TSFKW_MI,
+    /*N*/ TSFKW_NS,     -1,
+    /*P*/ TSFKW_PM,     -1, -1,
+    /*S*/ TSFKW_SS,
+    /*T*/ TSFKW_TZH,
+    /*U*/ TSFKW_US,     -1, -1, -1,
+    /*Y*/ TSFKW_YYYY,   -1,
+    /*[ \ ] ^ _ `*/ -1, -1, -1, -1, -1, -1,
+    /*a*/ TSFKW_am,     -1, -1,
+    /*d*/ TSFKW_day,    -1, -1, -1,
+    /*h*/ TSFKW_hh24,   -1, -1, -1, -1,
+    /*m*/ TSFKW_mi,
+    /*n*/ TSFKW_ns,     -1,
+    /*p*/ TSFKW_pm,     -1, -1,
+    /*s*/ TSFKW_ss,
+    /*t*/ TSFKW_tzh,
+    /*u*/ TSFKW_us,     -1, -1, -1,
+    /*y*/ TSFKW_yyyy,   -1};
+
+typedef struct {
+  uint8_t                type;
+  const char*            c;
+  int32_t                len;
+  const TSFormatKeyWord* key;
+} TSFormatNode;
+
+static const char* const weekDays[] = {"Sunday",   "Monday", "Tuesday",  "Wednesday",
+                                       "Thursday", "Friday", "Saturday", "NULL"};
+static const char* const shortWeekDays[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "NULL"};
+static const char* const fullMonths[] = {"January", "February",  "March",   "April",    "May",      "June", "July",
+                                         "August",  "September", "October", "November", "December", NULL};
+static const char* const months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul",
+                                     "Aug", "Sep", "Oct", "Nov", "Dec", NULL};
+#define A_M_STR "A.M."
+#define a_m_str "a.m."
+#define AM_STR  "AM"
+#define am_str  "am"
+#define P_M_STR "P.M."
+#define p_m_str "p.m."
+#define PM_STR  "PM"
+#define pm_str  "pm"
+static const char* const apms[] = {AM_STR, PM_STR, am_str, pm_str, NULL};
+static const char* const long_apms[] = {A_M_STR, P_M_STR, a_m_str, p_m_str, NULL};
+
+#define TS_FORMAT_NODE_TYPE_KEYWORD   1
+#define TS_FORMAT_NODE_TYPE_SEPARATOR 2
+#define TS_FORMAT_NODE_TYPE_CHAR      3
+
+static const TSFormatKeyWord* keywordSearch(const char* str) {
+  if (*str < 'A' || *str > 'z' || (*str > 'Z' && *str < 'a')) return NULL;
+  int32_t idx = TSFormatKeywordIndex[str[0] - 'A'];
+  if (idx < 0) return NULL;
+  const TSFormatKeyWord* key = &formatKeyWords[idx++];
+  while (key->name && str[0] == key->name[0]) {
+    if (0 == strncmp(key->name, str, key->len)) {
+      return key;
+    }
+    key = &formatKeyWords[idx++];
+  }
+  return NULL;
+}
+
+static bool isSeperatorChar(char c) {
+  return (c > 0x20 && c < 0x7F && !(c >= 'A' && c <= 'Z') && !(c >= 'a' && c <= 'z') && !(c >= '0' && c <= '9'));
+}
+
+static void parseTsFormat(const char* formatStr, SArray* formats) {
+  TSFormatNode* lastOtherFormat = NULL;
+  while (*formatStr) {
+    const TSFormatKeyWord* key = keywordSearch(formatStr);
+    if (key) {
+      TSFormatNode format = {.key = key, .type = TS_FORMAT_NODE_TYPE_KEYWORD};
+      taosArrayPush(formats, &format);
+      formatStr += key->len;
+      lastOtherFormat = NULL;
+    } else {
+      if (*formatStr == '"') {
+        lastOtherFormat = NULL;
+        // for double quoted string
+        formatStr++;
+        TSFormatNode* last = NULL;
+        while (*formatStr) {
+          if (*formatStr == '"') {
+            formatStr++;
+            break;
+          }
+          if (*formatStr == '\\' && *(formatStr + 1)) {
+            formatStr++;
+            last = NULL; // stop expanding last format, create new format
+          }
+          if (last) {
+            // expand
+            assert(last->type == TS_FORMAT_NODE_TYPE_CHAR);
+            last->len++;
+            formatStr++;
+          } else {
+            // create new
+            TSFormatNode format = {.type = TS_FORMAT_NODE_TYPE_CHAR, .key = NULL};
+            format.c = formatStr;
+            format.len = 1;
+            taosArrayPush(formats, &format);
+            formatStr++;
+            last = taosArrayGetLast(formats);
+          }
+        }
+      } else {
+        // for other strings
+        if (*formatStr == '\\' && *(formatStr + 1)) {
+          formatStr++;
+          lastOtherFormat = NULL; // stop expanding
+        } else {
+          if (lastOtherFormat && !isSeperatorChar(*formatStr)) {
+            // expanding
+          } else {
+            // create new
+            lastOtherFormat = NULL;
+          }
+        }
+        if (lastOtherFormat) {
+          assert(lastOtherFormat->type == TS_FORMAT_NODE_TYPE_CHAR);
+          lastOtherFormat->len++;
+          formatStr++;
+        } else {
+          TSFormatNode format = {
+            .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR,
+            .key = NULL};
+          format.c = formatStr;
+          format.len = 1;
+          taosArrayPush(formats, &format);
+          formatStr++;
+          if (format.type == TS_FORMAT_NODE_TYPE_CHAR) lastOtherFormat = taosArrayGetLast(formats);
+        }
+      }
+    }
+  }
+}
+
+static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
+  int32_t size = taosArrayGetSize(formats);
+  const char* start = s;
+  for (int32_t i = 0; i < size; ++i) {
+    TSFormatNode* format = taosArrayGet(formats, i);
+    if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) {
+      if (s - start + format->len + 1 > outLen) break;
+      strncpy(s, format->c, format->len);
+      s += format->len;
+      continue;
+    }
+    if (s - start + 16 > outLen) break;
+
+    switch (format->key->id) {
+      case TSFKW_AM:
+      case TSFKW_PM:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "PM" : "AM");
+        s += 2;
+        break;
+      case TSFKW_A_M:
+      case TSFKW_P_M:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "P.M." : "A.M.");
+        s += 4;
+        break;
+      case TSFKW_am:
+      case TSFKW_pm:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "pm" : "am");
+        s += 2;
+        break;
+      case TSFKW_a_m:
+      case TSFKW_p_m:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "p.m." : "a.m.");
+        s += 4;
+        break;
+      case TSFKW_DDD:
+        sprintf(s, "%d", tm->tm.tm_yday);
+        s += strlen(s);
+        break;
+      case TSFKW_DD:
+        sprintf(s, "%02d", tm->tm.tm_mday);
+        s += 2;
+        break;
+      case TSFKW_D:
+        sprintf(s, "%d", tm->tm.tm_wday + 1);
+        s += 1;
+        break;
+      case TSFKW_DAY: {
+        // MONDAY, TUESDAY...
+        const char* wd = weekDays[tm->tm.tm_wday];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_Day:
+        // Monday, TuesDay...
+        sprintf(s, "%-9s", weekDays[tm->tm.tm_wday]);
+        s += strlen(s);
+        break;
+      case TSFKW_day: {
+        const char* wd = weekDays[tm->tm.tm_wday];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_DY: {
+        // MON, TUE
+        const char* wd = shortWeekDays[tm->tm.tm_wday];
+        char        buf[8] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]);
+        sprintf(s, "%3s", buf);
+        s += 3;
+        break;
+      }
+      case TSFKW_Dy:
+        // Mon, Tue
+        sprintf(s, "%3s", shortWeekDays[tm->tm.tm_wday]);
+        s += 3;
+        break;
+      case TSFKW_dy: {
+        // mon, tue
+        const char* wd = shortWeekDays[tm->tm.tm_wday];
+        char        buf[8] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]);
+        sprintf(s, "%3s", buf);
+        s += 3;
+        break;
+      }
+      case TSFKW_HH24:
+        sprintf(s, "%02d", tm->tm.tm_hour);
+        s += 2;
+        break;
+      case TSFKW_HH:
+      case TSFKW_HH12:
+        // 0 or 12 o'clock in 24H coresponds to 12 o'clock (AM/PM) in 12H
+        sprintf(s, "%02d", tm->tm.tm_hour % 12 == 0 ? 12 : tm->tm.tm_hour % 12);
+        s += 2;
+        break;
+      case TSFKW_MI:
+        sprintf(s, "%02d", tm->tm.tm_min);
+        s += 2;
+        break;
+      case TSFKW_MM:
+        sprintf(s, "%02d", tm->tm.tm_mon + 1);
+        s += 2;
+        break;
+      case TSFKW_MONTH: {
+        const char* mon = fullMonths[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_MON: {
+        const char* mon = months[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]);
+        sprintf(s, "%s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_Month:
+        sprintf(s, "%-9s", fullMonths[tm->tm.tm_mon]);
+        s += strlen(s);
+        break;
+      case TSFKW_month: {
+        const char* mon = fullMonths[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_Mon:
+        sprintf(s, "%s", months[tm->tm.tm_mon]);
+        s += strlen(s);
+        break;
+      case TSFKW_mon: {
+        const char* mon = months[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]);
+        sprintf(s, "%s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_SS:
+        sprintf(s, "%02d", tm->tm.tm_sec);
+        s += 2;
+        break;
+      case TSFKW_MS:
+        sprintf(s, "%03" PRId64, tm->fsec / 1000000L);
+        s += 3;
+        break;
+      case TSFKW_US:
+        sprintf(s, "%06" PRId64, tm->fsec / 1000L);
+        s += 6;
+        break;
+      case TSFKW_NS:
+        sprintf(s, "%09" PRId64, tm->fsec);
+        s += 9;
+        break;
+      case TSFKW_TZH:
+        sprintf(s, "%s%02d", tsTimezone < 0 ? "-" : "+", tsTimezone);
+        s += strlen(s);
+        break;
+      case TSFKW_YYYY:
+        sprintf(s, "%04d", tm->tm.tm_year + 1900);
+        s += strlen(s);
+        break;
+      case TSFKW_YYY:
+        sprintf(s, "%03d", (tm->tm.tm_year + 1900) % 1000);
+        s += strlen(s);
+        break;
+      case TSFKW_YY:
+        sprintf(s, "%02d", (tm->tm.tm_year + 1900) % 100);
+        s += strlen(s);
+        break;
+      case TSFKW_Y:
+        sprintf(s, "%01d", (tm->tm.tm_year + 1900) % 10);
+        s += strlen(s);
+        break;
+      default:
+        break;
+    }
+  }
+}
+
+/// @brief find s in arr case insensitively
+/// @retval the index in arr if found, -1 if not found
+static int32_t strArrayCaseSearch(const char* const* arr, const char* s) {
+  if (!*s) return -1;
+  const char* const* fmt = arr;
+  for (; *fmt; ++fmt) {
+    const char *l, *r;
+    for (l = fmt[0], r = s;; l++, r++) {
+      if (*l == '\0') return fmt - arr;
+      if (*r == '\0' || tolower(*l) != tolower(*r)) break;
+    }
+  }
+  return -1;
+}
+
+static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len, bool needMoreDigit) {
+  char*       last;
+  int64_t     res;
+  const char* s = str;
+  if ('\0' == str[0]) return NULL;
+  if (len <= 0) {
+    res = taosStr2Int64(s, &last, 10);
+    s = last;
+  } else {
+    char buf[16] = {0};
+    strncpy(buf, s, len);
+    int32_t copiedLen = strlen(buf);
+    if (copiedLen < len) {
+      if (!needMoreDigit) {
+        // digits not enough, that's ok, cause we do not need more digits
+        // '2023-1' 'YYYY-MM'
+        // '202a' 'YYYY' -> 202
+        res = taosStr2Int64(s, &last, 10);
+        s += copiedLen;
+      } else {
+        // bytes not enough, and there are other digit formats to match
+        // '2023-1' 'YYYY-MMDD'
+        return NULL;
+      }
+    } else {
+      if (needMoreDigit) {
+        res = taosStr2Int64(buf, &last, 10);
+        // bytes enough, but digits not enough, like '202a12' 'YYYYMM', YYYY needs four digits
+        if (last - buf < len) return NULL;
+        s += last - buf;
+      } else {
+        res = taosStr2Int64(s, &last, 10);
+        s = last;
+      }
+    }
+  }
+  if (s == str) {
+    // no integers found
+    return NULL;
+  }
+  if (errno == ERANGE || res > INT32_MAX || res < INT32_MIN) {
+    // out of range
+    return NULL;
+  }
+  *dest = res;
+  return s;
+}
+
+static int32_t adjustYearTo2020(int32_t year) {
+  if (year < 70) return year + 2000;    // 2000 - 2069
+  if (year < 100) return year + 1900;   // 1970 - 1999
+  if (year < 520) return year + 2000;   // 2100 - 2519
+  if (year < 1000) return year + 1000;  // 1520 - 1999
+  return year;
+}
+
+static bool checkTm(const struct tm* tm) {
+  if (tm->tm_mon < 0 || tm->tm_mon > 11) return false;
+  if (tm->tm_wday < 0 || tm->tm_wday > 6) return false;
+  if (tm->tm_yday < 0 || tm->tm_yday > 365) return false;
+  if (tm->tm_mday < 0 || tm->tm_mday > 31) return false;
+  if (tm->tm_hour < 0 || tm->tm_hour > 23) return false;
+  if (tm->tm_min < 0 || tm->tm_min > 59) return false;
+  if (tm->tm_sec < 0 || tm->tm_sec > 60) return false;
+  return true;
+}
+
+static bool needMoreDigits(SArray* formats, int32_t curIdx) {
+  if (curIdx == taosArrayGetSize(formats) - 1) return false;
+  TSFormatNode* pNextNode = taosArrayGet(formats, curIdx + 1);
+  if (pNextNode->type == TS_FORMAT_NODE_TYPE_SEPARATOR) {
+    return false;
+  } else if (pNextNode->type == TS_FORMAT_NODE_TYPE_KEYWORD) {
+    return pNextNode->key->isDigit;
+  } else {
+    return isdigit(pNextNode->c[0]);
+  }
+}
+
+/// @brief convert a formatted time str to timestamp
+/// @param[in] s the formatted timestamp str
+/// @param[in] formats array of TSFormatNode, output of parseTsFormat
+/// @param[out] ts output timestamp
+/// @param precision the timestamp precision to convert to, sec/milli/micro/nano
+/// @param[out] sErrPos if not NULL, when err occured, points to the failed position of s, only set when ret is -1
+/// @param[out] fErrIdx if not NULL, when err occured, the idx of the failed format idx, only set when ret is -1
+/// @retval 0 for success
+/// @retval -1 for format and s mismatch error
+/// @retval -2 if datetime err, like 2023-13-32 25:61:69
+static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t precision, const char** sErrPos,
+                       int32_t* fErrIdx) {
+  int32_t size = taosArrayGetSize(formats);
+  int32_t pm = 0;      // default am
+  int32_t hour12 = 0;  // default HH24
+  int32_t year = 0, mon = 0, yd = 0, md = 1, wd = 0;
+  int32_t hour = 0, min = 0, sec = 0, us = 0, ms = 0, ns = 0;
+  int32_t tzSign = 1, tz = tsTimezone;
+  int32_t err = 0;
+
+  for (int32_t i = 0; i < size && *s != '\0'; ++i) {
+    while (isspace(*s) && *s != '\0') {
+      s++;
+    }
+    if (!s) break;
+    TSFormatNode* node = taosArrayGet(formats, i);
+    if (node->type == TS_FORMAT_NODE_TYPE_SEPARATOR) {
+      // separator matches any character
+      if (isSeperatorChar(s[0])) s += node->len;
+      continue;
+    }
+    if (node->type == TS_FORMAT_NODE_TYPE_CHAR) {
+      int32_t pos = 0;
+      // skip leading spaces
+      while (isspace(node->c[pos]) && node->len > 0) pos++;
+      while (pos < node->len && *s != '\0') {
+        if (!isspace(node->c[pos++])) {
+          while (isspace(*s) && *s != '\0') s++;
+          if (*s != '\0') s++;  // forward together
+        }
+      }
+      continue;
+    }
+    assert(node->type == TS_FORMAT_NODE_TYPE_KEYWORD);
+    switch (node->key->id) {
+      case TSFKW_A_M:
+      case TSFKW_P_M:
+      case TSFKW_a_m:
+      case TSFKW_p_m: {
+        int32_t idx = strArrayCaseSearch(long_apms, s);
+        if (idx >= 0) {
+          s += 4;
+          pm = idx % 2;
+          hour12 = 1;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_AM:
+      case TSFKW_PM:
+      case TSFKW_am:
+      case TSFKW_pm: {
+        int32_t idx = strArrayCaseSearch(apms, s);
+        if (idx >= 0) {
+          s += 2;
+          pm = idx % 2;
+          hour12 = 1;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_HH:
+      case TSFKW_HH12: {
+        const char* newPos = tsFormatStr2Int32(&hour, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos || hour > 12 || hour <= 0) {
+          err = -1;
+        } else {
+          hour12 = 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_HH24: {
+        const char* newPos = tsFormatStr2Int32(&hour, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          hour12 = 0;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_MI: {
+        const char* newPos = tsFormatStr2Int32(&min, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_SS: {
+        const char* newPos = tsFormatStr2Int32(&sec, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else
+          s = newPos;
+      } break;
+      case TSFKW_MS: {
+        const char* newPos = tsFormatStr2Int32(&ms, s, 3, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          int32_t len = newPos - s;
+          ms *= len == 1 ? 100 : len == 2 ? 10 : 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_US: {
+        const char* newPos = tsFormatStr2Int32(&us, s, 6, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          int32_t len = newPos - s;
+          us *= len == 1 ? 100000 : len == 2 ? 10000 : len == 3 ? 1000 : len == 4 ? 100 : len == 5 ? 10 : 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_NS: {
+        const char* newPos = tsFormatStr2Int32(&ns, s, 9, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          int32_t len = newPos - s;
+          ns *= len == 1   ? 100000000
+                : len == 2 ? 10000000
+                : len == 3 ? 1000000
+                : len == 4 ? 100000
+                : len == 5 ? 10000
+                : len == 6 ? 1000
+                : len == 7 ? 100
+                : len == 8 ? 10
+                           : 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_TZH: {
+        tzSign = *s == '-' ? -1 : 1;
+        const char* newPos = tsFormatStr2Int32(&tz, s, -1, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_MONTH:
+      case TSFKW_Month:
+      case TSFKW_month: {
+        int32_t idx = strArrayCaseSearch(fullMonths, s);
+        if (idx >= 0) {
+          s += strlen(fullMonths[idx]);
+          mon = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_MON:
+      case TSFKW_Mon:
+      case TSFKW_mon: {
+        int32_t idx = strArrayCaseSearch(months, s);
+        if (idx >= 0) {
+          s += strlen(months[idx]);
+          mon = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_MM: {
+        const char* newPos = tsFormatStr2Int32(&mon, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+          mon -= 1;
+        }
+      } break;
+      case TSFKW_DAY:
+      case TSFKW_Day:
+      case TSFKW_day: {
+        int32_t idx = strArrayCaseSearch(weekDays, s);
+        if (idx >= 0) {
+          s += strlen(weekDays[idx]);
+          wd = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_DY:
+      case TSFKW_Dy:
+      case TSFKW_dy: {
+        int32_t idx = strArrayCaseSearch(shortWeekDays, s);
+        if (idx >= 0) {
+          s += strlen(shortWeekDays[idx]);
+          wd = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_DDD: {
+        const char* newPos = tsFormatStr2Int32(&yd, s, 3, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_DD: {
+        const char* newPos = tsFormatStr2Int32(&md, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_D: {
+        const char* newPos = tsFormatStr2Int32(&wd, s, 1, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_YYYY: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 4, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_YYY: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 3, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          year = adjustYearTo2020(year);
+          s = newPos;
+        }
+      } break;
+      case TSFKW_YY: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          year = adjustYearTo2020(year);
+          s = newPos;
+        }
+      } break;
+      case TSFKW_Y: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 1, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          year = adjustYearTo2020(year);
+          s = newPos;
+        }
+      } break;
+      default:
+        break;
+    }
+    if (err) {
+      if (sErrPos) *sErrPos = s;
+      if (fErrIdx) *fErrIdx = i;
+      return err;
+    }
+  }
+  struct STm tm = {0};
+  tm.tm.tm_year = year - 1900;
+  tm.tm.tm_mon = mon;
+  tm.tm.tm_yday = yd;
+  tm.tm.tm_mday = md;
+  tm.tm.tm_wday = wd;
+  if (hour12) {
+    if (pm && hour < 12)
+      tm.tm.tm_hour = hour + 12;
+    else if (!pm && hour == 12)
+      tm.tm.tm_hour = 0;
+    else
+      tm.tm.tm_hour = hour;
+  } else {
+    tm.tm.tm_hour = hour;
+  }
+  tm.tm.tm_min = min;
+  tm.tm.tm_sec = sec;
+  if (!checkTm(&tm.tm)) return -2;
+  if (tz < -12 || tz > 12) return -2;
+  tm.fsec = ms * 1000000 + us * 1000 + ns;
+  int32_t ret = taosTm2Ts(&tm, ts, precision);
+  *ts += 60 * 60 * (tsTimezone - tz) * TICK_PER_SECOND[precision];
+  return ret;
+}
+
+void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) {
+  if (!*formats) {
+    *formats = taosArrayInit(8, sizeof(TSFormatNode));
+    parseTsFormat(format, *formats);
+  }
+  struct STm tm;
+  taosTs2Tm(ts, precision, &tm);
+  tm2char(*formats, &tm, out, outLen);
+}
+
+int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg,
+                    int32_t errMsgLen) {
+  const char* sErrPos;
+  int32_t     fErrIdx;
+  if (!*formats) {
+    *formats = taosArrayInit(4, sizeof(TSFormatNode));
+    parseTsFormat(format, *formats);
+  }
+  int32_t code = char2ts(tsStr, *formats, ts, precision, &sErrPos, &fErrIdx);
+  if (code == -1) {
+    TSFormatNode* fNode = (taosArrayGet(*formats, fErrIdx));
+    snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos,
+             fErrIdx < taosArrayGetSize(*formats) ? ((TSFormatNode*)taosArrayGet(*formats, fErrIdx))->key->name : "");
+  } else if (code == -2) {
+    snprintf(errMsg, errMsgLen, "timestamp format error: %s -> %s", tsStr, format);
+  }
+  return code;
+}
+
+void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen) {
+  SArray* formats = taosArrayInit(4, sizeof(TSFormatNode));
+  parseTsFormat(format, formats);
+  struct STm tm;
+  taosTs2Tm(ts, precision, &tm);
+  tm2char(formats, &tm, out, outLen);
+  taosArrayDestroy(formats);
+}
+
+int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr) {
+  const char* sErrPos;
+  int32_t     fErrIdx;
+  SArray*     formats = taosArrayInit(4, sizeof(TSFormatNode));
+  parseTsFormat(format, formats);
+  int32_t code = char2ts(tsStr, formats, ts, precision, &sErrPos, &fErrIdx);
+  if (code == -1) {
+    printf("failed position: %s\n", sErrPos);
+    printf("failed format: %s\n", ((TSFormatNode*)taosArrayGet(formats, fErrIdx))->key->name);
+  }
+  taosArrayDestroy(formats);
+  return code;
+}
--- a/source/common/test/commonTests.cpp
+++ b/source/common/test/commonTests.cpp
@ -13,6 +13,7 @@
 #include "tdatablock.h"
 #include "tdef.h"
 #include "tvariant.h"
+#include "ttime.h"

 namespace {
 //
@ -260,4 +261,234 @@ TEST(testCase, var_dataBlock_split_test) {
  }
 }

+void check_tm(const STm* tm, int32_t y, int32_t mon, int32_t d, int32_t h, int32_t m, int32_t s, int64_t fsec) {
+  ASSERT_EQ(tm->tm.tm_year, y);
+  ASSERT_EQ(tm->tm.tm_mon, mon);
+  ASSERT_EQ(tm->tm.tm_mday, d);
+  ASSERT_EQ(tm->tm.tm_hour, h);
+  ASSERT_EQ(tm->tm.tm_min, m);
+  ASSERT_EQ(tm->tm.tm_sec, s);
+  ASSERT_EQ(tm->fsec, fsec);
+}
+
+void test_timestamp_tm_conversion(int64_t ts, int32_t precision, int32_t y, int32_t mon, int32_t d, int32_t h, int32_t m, int32_t s, int64_t fsec) {
+  int64_t ts_tmp;
+  char buf[128] = {0};
+  struct STm tm;
+  taosFormatUtcTime(buf, 128, ts, precision);
+  printf("formated ts of %ld, precision: %d is: %s\n", ts, precision, buf);
+  taosTs2Tm(ts, precision, &tm);
+  check_tm(&tm, y, mon, d, h, m, s, fsec);
+  taosTm2Ts(&tm, &ts_tmp, precision);
+  ASSERT_EQ(ts, ts_tmp);
+}
+
+TEST(timeTest, timestamp2tm) {
+  const char* ts_str_ns = "2023-10-12T11:29:00.775726171+0800";
+  const char* ts_str_us = "2023-10-12T11:29:00.775726+0800";
+  const char* ts_str_ms = "2023-10-12T11:29:00.775+0800";
+  int64_t     ts, tmp_ts = 0;
+  struct STm  tm;
+
+  ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ns, &ts, strlen(ts_str_ns), TSDB_TIME_PRECISION_NANO, 0));
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_NANO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0,
+                               775726171L);
+
+  ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_us, &ts, strlen(ts_str_us), TSDB_TIME_PRECISION_MICRO, 0));
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MICRO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0,
+                               775726000L);
+
+  ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ms, &ts, strlen(ts_str_ms), TSDB_TIME_PRECISION_MILLI, 0));
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0,
+                               775000000L);
+
+  ts = -5364687943000;  // milliseconds since epoch, Wednesday, January 1, 1800 1:00:00 AM GMT+08:06
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 1800 - 1900, 0 /* mon start from 0*/, 1, 1, 0, 0,
+                               000000000L);
+
+  ts = 0;
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 1970 - 1900, 0 /* mon start from 0*/, 1, 8, 0, 0,
+                               000000000L);
+
+  ts = -62198784343000; // milliseconds before epoch, Friday, January 1, -0001 12:00:00 AM GMT+08:06
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, -1 - 1900, 0 /* mon start from 0*/, 1,
+                               0 /* hour start from 0*/, 0, 0, 000000000L);
+}
+
+void test_ts2char(int64_t ts, const char* format, int32_t precison, const char* expected) {
+  char buf[256] = {0};
+  TEST_ts2char(format, ts, precison, buf, 256);
+  printf("ts: %ld format: %s res: [%s], expected: [%s]\n", ts, format, buf, expected);
+  ASSERT_STREQ(expected, buf);
+}
+
+TEST(timeTest, ts2char) {
+  osDefaultInit();
+  if (tsTimezone != TdEastZone8) GTEST_SKIP();
+  int64_t ts;
+  const char* format = "YYYY-MM-DD";
+  ts = 0;
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_MILLI, "1970-01-01");
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_MICRO, "1970-01-01");
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_NANO, "1970-01-01");
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_SECONDS, "1970-01-01");
+
+  ts = 1697163517;
+  test_ts2char(ts, "YYYY-MM-DD", TSDB_TIME_PRECISION_SECONDS, "2023-10-13");
+  ts = 1697163517000;
+  test_ts2char(ts, "YYYY-MM-DD-Day-DAY", TSDB_TIME_PRECISION_MILLI, "2023-10-13-Friday   -FRIDAY   ");
+#ifndef WINDOWS
+  // double quoted: year, month, day are not parsed
+  test_ts2char(ts,
+               "YYYY-YYY-YY-Y-yyyy-yyy-yy-y-\"年\"-MONTH-MON-Month-Mon-month-mon-\"月\"-DDD-DD-D-ddd-dd-d-DAY-Day-"
+               "day-\"日\"",
+               TSDB_TIME_PRECISION_MILLI,
+               "2023-023-23-3-2023-023-23-3-年-OCTOBER  -OCT-October  -Oct-october  "
+               "-oct-月-285-13-6-285-13-6-FRIDAY   -Friday   -friday   -日");
+#endif
+  ts = 1697182085123L; // Friday, October 13, 2023 3:28:05.123 PM GMT+08:00
+  test_ts2char(ts, "HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI,
+               "15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm");
+
+  // double quotes normal output
+  test_ts2char(ts, "\\\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am\\\"", TSDB_TIME_PRECISION_MILLI,
+               "\"15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm\"");
+  test_ts2char(ts, "\\\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI,
+               "\"15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm");
+  // double quoted strings recognized as literal string, parsing skipped
+  test_ts2char(ts, "\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI,
+               "HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am");
+  test_ts2char(ts, "yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "2023-10-13 15:28:05.123000000pmaaa");
+  test_ts2char(ts, "aaa--yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "aaa--2023-10-13 15:28:05.123000000pmaaa");
+  test_ts2char(ts, "add--yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "a13--2023-10-13 15:28:05.123000000pmaaa");
+
+  ts = 1693946405000;
+  test_ts2char(ts, "Day, Month dd, YYYY hh24:mi:ss AM TZH:tzh", TSDB_TIME_PRECISION_MILLI, "Wednesday, September 06, 2023 04:40:05 AM +08:+08");
+
+  ts = -62198784343000; // milliseconds before epoch, Friday, January 1, -0001 12:00:00 AM GMT+08:06
+  test_ts2char(ts, "Day, Month dd, YYYY hh12:mi:ss AM", TSDB_TIME_PRECISION_MILLI, "Friday   , January   01, -001 12:00:00 AM");
+}
+
+TEST(timeTest, char2ts) {
+  osDefaultInit();
+  if (tsTimezone != TdEastZone8) GTEST_SKIP();
+  int64_t ts;
+  int32_t code =
+      TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM");
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(ts, 1696867200000LL);
+
+  // 2009-1-1 00:00:00
+  ASSERT_EQ(0, TEST_char2ts("YYYY-YYY-YY-Y", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9"));
+  ASSERT_EQ(1230739200000LL, ts);
+  // 2023-1-1
+  ASSERT_EQ(0, TEST_char2ts("YYYY-YYY-YY", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9"));
+  ASSERT_EQ(ts, 1672502400000LL);
+
+  // 2123-1-1, the second year(123) is used, which converted to 2123
+  ASSERT_EQ(0, TEST_char2ts("YYYY-YYY", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9"));
+  ASSERT_EQ(ts, 4828176000000LL);
+  // 2023-1-1 12:10:10am
+  ASSERT_EQ(0, TEST_char2ts("yyyy-mm-dd HH12:MI:SSAM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-1-1 12:10:10am"));
+  ASSERT_EQ(ts, 1672503010000LL);
+
+  // 2023-1-1 21:10:10.123
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH12:MI:ss.msa.m.", &ts, TSDB_TIME_PRECISION_MILLI, "23-1-01 9:10:10.123p.m."));
+  ASSERT_EQ(ts, 1672578610123LL);
+
+  // 2023-1-1 21:10:10.123456789
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH:MI:ss.ms.us.nsa.m.", &ts, TSDB_TIME_PRECISION_NANO,
+                            "23-1-01 9:10:10.123.000456.000000789p.m."));
+  ASSERT_EQ(ts, 1672578610123456789LL);
+
+  // 2023-1-1 21:10:10.120450780
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH24:MI:SS.ms.us.ns", &ts, TSDB_TIME_PRECISION_NANO,
+                            "   23   - 1 - 01   \t  21:10:10 .  12 .  \t 00045 . 00000078  \t"));
+  ASSERT_EQ(ts, 1672578610120450780LL);
+
+#ifndef WINDOWS
+  // 2023-1-1 21:10:10.120450780
+  ASSERT_EQ(0, TEST_char2ts("yy \"年\"-MM 月-dd  \"日 子\" HH24:MI:ss.ms.us.ns TZH", &ts, TSDB_TIME_PRECISION_NANO,
+                            "   23  年 - 1 月 - 01 日 子  \t  21:10:10 .  12 .  \t 00045 . 00000078  \t+08"));
+  ASSERT_EQ(ts, 1672578610120450780LL);
+#endif
+
+  // 2023-1-1 19:10:10.123456789+06 -> 2023-1-1 21:10:10.123456789+08
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH:MI:ss.ms.us.nsa.m.TZH", &ts, TSDB_TIME_PRECISION_NANO,
+                            "23-1-01 7:10:10.123.000456.000000789p.m.6"));
+  ASSERT_EQ(ts, 1672578610123456789LL);
+
+  // 2023-1-1 12:10:10.123456789-01 -> 2023-1-1 21:10:10.123456789+08
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH24:MI:ss.ms.us.nsTZH", &ts, TSDB_TIME_PRECISION_NANO,
+                            "23-1-01 12:10:10.123.000456.000000789-1"));
+  ASSERT_EQ(ts, 1672578610123456789LL);
+
+  // 2100-01-01 11:10:10.124456+08
+  ASSERT_EQ(
+      0, TEST_char2ts("yyyy-MM-dd HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, "2100-01-01 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  // 2100-01-01 11:10:10.124456+08 Firday
+  ASSERT_EQ(0, TEST_char2ts("yyyy/MONTH/dd DAY HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 friday 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyy/Month/dd Day HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 FRIDAY 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+  ASSERT_EQ(0, TEST_char2ts("yyyy/Month/dd Dy HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 Fri 11:10:10.124456+08:00"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyy/month/dd day HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 Friday 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  // 2100-02-01 11:10:10.124456+08 Firday
+  ASSERT_EQ(0, TEST_char2ts("yyyy/mon/dd DY HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/Feb/01 Mon 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4105134610124456LL);
+
+  // 2100-02-01 11:10:10.124456+08 Firday
+  ASSERT_EQ(0, TEST_char2ts("yyyy/mon/dd DY DDD-DD-D HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/Feb/01 Mon 100-1-01 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4105134610124456LL);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "21000101"));
+
+  // What is Fe?
+  ASSERT_EQ(-1, TEST_char2ts("yyyy/mon/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100/Fe/01"));
+  // '/' cannot convert to MM
+  ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100/2/1"));
+  // nothing to be converted to dd
+  ASSERT_EQ(0, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "210012"));
+  ASSERT_EQ(ts, 4131273600000000LL); // 2100-12-1
+  ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "21001"));
+  ASSERT_EQ(-1, TEST_char2ts("yyyyMM-dd ", &ts, TSDB_TIME_PRECISION_MICRO, "23a1-1"));
+
+  // 2100-1-2
+  ASSERT_EQ(0, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "21001/2"));
+  ASSERT_EQ(ts, 4102502400000000LL);
+
+  // default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00
+  ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970"));
+  ASSERT_EQ(ts, -1 * tsTimezone * 60 * 60);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2"));
+  ASSERT_EQ(ts, 4102502400000000LL);
+
+  ASSERT_EQ(-2, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210013/2"));
+  ASSERT_EQ(-2, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210011/32"));
+  ASSERT_EQ(-1, TEST_char2ts("HH12:MI:SS", &ts, TSDB_TIME_PRECISION_MICRO, "21:12:12"));
+  ASSERT_EQ(-1, TEST_char2ts("yyyy/MM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100111111111/11/2"));
+  ASSERT_EQ(-2, TEST_char2ts("yyyy/MM1/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "23/11/2-13"));
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年1/1+0"));
+  ASSERT_EQ(ts, 0);
+  ASSERT_EQ(-1, TEST_char2ts("yyyy年a MM/dd", &ts, TSDB_TIME_PRECISION_MICRO, "2023年1/2"));
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年   1/1+0"));
+  ASSERT_EQ(ts, 0);
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a a a 1/1+0"));
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a a a a a a a a a a a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a     "));
+}
+
 #pragma GCC diagnostic pop
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@ -163,7 +163,7 @@ SArray *mmGetMsgHandles() {
  if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
  if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
  if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
-
+  if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

  if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
  if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@ -137,7 +137,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
    SRetention *pRetention = &pCfg->tsdbCfg.retentions[i];
    memcpy(pRetention, taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention));
    if (i == 0) {
-      if ((pRetention->freq > 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
+      if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
    }
  }

--- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
+++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
@ -97,7 +97,7 @@ int32_t       dmMarkWrapper(SMgmtWrapper *pWrapper);
 void          dmReleaseWrapper(SMgmtWrapper *pWrapper);
 int32_t       dmInitVars(SDnode *pDnode);
 void          dmClearVars(SDnode *pDnode);
-int32_t       dmInitModule(SDnode *pDnode);
+int32_t       dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
 bool          dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
 SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
 void          dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
@ -119,7 +119,7 @@ int32_t dmInitStatusClient(SDnode *pDnode);
 void    dmCleanupClient(SDnode *pDnode);
 void    dmCleanupStatusClient(SDnode *pDnode);
 SMsgCb  dmGetMsgcb(SDnode *pDnode);
-int32_t dmInitMsgHandle(SDnode *pDnode);
+int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
 int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

 // dmMonitor.c
--- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
@ -66,7 +66,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
    goto _OVER;
  }

-  if (dmInitModule(pDnode) != 0) {
+  if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
    goto _OVER;
  }

@ -107,6 +107,75 @@ void dmCleanupDnode(SDnode *pDnode) {
  dDebug("dnode is closed, ptr:%p", pDnode);
 }

+int32_t dmInitVars(SDnode *pDnode) {
+  SDnodeData *pData = &pDnode->data;
+  pData->dnodeId = 0;
+  pData->clusterId = 0;
+  pData->dnodeVer = 0;
+  pData->engineVer = 0;
+  pData->updateTime = 0;
+  pData->rebootTime = taosGetTimestampMs();
+  pData->dropped = 0;
+  pData->stopped = 0;
+
+  pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+  if (pData->dnodeHash == NULL) {
+    dError("failed to init dnode hash");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (dmReadEps(pData) != 0) {
+    dError("failed to read file since %s", terrstr());
+    return -1;
+  }
+
+  if (pData->dropped) {
+    dError("dnode will not start since its already dropped");
+    return -1;
+  }
+
+  taosThreadRwlockInit(&pData->lock, NULL);
+  taosThreadMutexInit(&pDnode->mutex, NULL);
+  return 0;
+}
+
+void dmClearVars(SDnode *pDnode) {
+  for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+    taosMemoryFreeClear(pWrapper->path);
+    taosThreadRwlockDestroy(&pWrapper->lock);
+  }
+  if (pDnode->lockfile != NULL) {
+    taosUnLockFile(pDnode->lockfile);
+    taosCloseFile(&pDnode->lockfile);
+    pDnode->lockfile = NULL;
+  }
+
+  SDnodeData *pData = &pDnode->data;
+  taosThreadRwlockWrlock(&pData->lock);
+  if (pData->oldDnodeEps != NULL) {
+    if (dmWriteEps(pData) == 0) {
+      dmRemoveDnodePairs(pData);
+    }
+    taosArrayDestroy(pData->oldDnodeEps);
+    pData->oldDnodeEps = NULL;
+  }
+  if (pData->dnodeEps != NULL) {
+    taosArrayDestroy(pData->dnodeEps);
+    pData->dnodeEps = NULL;
+  }
+  if (pData->dnodeHash != NULL) {
+    taosHashCleanup(pData->dnodeHash);
+    pData->dnodeHash = NULL;
+  }
+  taosThreadRwlockUnlock(&pData->lock);
+
+  taosThreadRwlockDestroy(&pData->lock);
+  taosThreadMutexDestroy(&pDnode->mutex);
+  memset(&pDnode->mutex, 0, sizeof(pDnode->mutex));
+}
+
 void dmSetStatus(SDnode *pDnode, EDndRunStatus status) {
  if (pDnode->status != status) {
    dDebug("dnode status set from %s to %s", dmStatStr(pDnode->status), dmStatStr(status));
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@ -251,11 +251,11 @@ _OVER:
  dmReleaseWrapper(pWrapper);
 }

-int32_t dmInitMsgHandle(SDnode *pDnode) {
+int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
  SDnodeTrans *pTrans = &pDnode->trans;

  for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+    SMgmtWrapper *pWrapper = wrappers + ntype;
    SArray       *pArray = (*pWrapper->func.getHandlesFp)();
    if (pArray == NULL) return -1;

--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@ -397,6 +397,7 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
    SDnodeInfo dInfo;
    dInfo.id = pDnode->id;
    dInfo.ep.port = pDnode->port;
+    dInfo.offlineReason = pDnode->offlineReason;
    tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
    tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
    tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
@ -781,11 +782,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg

    SDnodeObj tmpDnode = *pDnode;
    if (action == DND_ACTIVE_CODE) {
-#ifndef TD_GRANT_OPTIMIZE
-      if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
-#else
      if (grantAlterActiveCode(pDnode->id, pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
-#endif
        if (TSDB_CODE_DUP_KEY != terrno) {
          mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
                 pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
@ -801,11 +798,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
        goto _OVER;
      }
    } else if (action == DND_CONN_ACTIVE_CODE) {
-#ifndef TD_GRANT_OPTIMIZE
-      if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
-#else
      if (grantAlterActiveCode(pDnode->id, pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
-#endif
        if (TSDB_CODE_DUP_KEY != terrno) {
          mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
                 pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
--- a/source/dnode/mnode/impl/src/mndGrant.c
+++ b/source/dnode/mnode/impl/src/mndGrant.c
@ -131,13 +131,9 @@ void    grantAdd(EGrantType grant, uint64_t value) {}
 void    grantRestore(EGrantType grant, uint64_t value) {}
 int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
 int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
-#ifndef TD_GRANT_OPTIMIZE
-int32_t grantAlterActiveCode(const char *old, const char *new, char *out, int8_t type) { return TSDB_CODE_SUCCESS; }
-#else
 int32_t grantAlterActiveCode(int32_t did, const char *old, const char *new, char *out, int8_t type) {
  return TSDB_CODE_SUCCESS;
 }
-#endif

 #endif

--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@ -281,7 +281,7 @@ static void *mndThreadFp(void *param) {
      mndCalMqRebalance(pMnode);
    }

-    if (sec % tsStreamCheckpointTickInterval == 0) {
+    if (sec % tsStreamCheckpointInterval == 0) {
      mndStreamCheckpointTick(pMnode, sec);
    }

--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@ -569,6 +569,10 @@ static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStr
 }

 static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) {
+  if (taosArrayGetSize(pTasksList) < SINK_NODE_LEVEL || pUpstreamTask == NULL) {
+    return;
+  }
+
  SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL);
  for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) {
    SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i);
@ -628,7 +632,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
    }

    setSinkTaskUpstreamInfo(pStream->tasks, pAggTask);
+    if (pHAggTask != NULL) {
      setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask);
+    }

    // source level
    return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, nextWindowSkey);
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@ -42,14 +42,14 @@ typedef struct SNodeEntry {
  int64_t hbTimestamp;  // second
 } SNodeEntry;

-typedef struct SStreamExecNodeInfo {
+typedef struct SStreamExecInfo {
  SArray       *pNodeEntryList;
  int64_t       ts;                // snapshot ts
  int64_t       activeCheckpoint;  // active check point id
  SHashObj     *pTaskMap;
  SArray       *pTaskList;
  TdThreadMutex lock;
-} SStreamExecNodeInfo;
+} SStreamExecInfo;

 typedef struct SVgroupChangeInfo {
  SHashObj *pDBMap;
@ -57,7 +57,7 @@ typedef struct SVgroupChangeInfo {
 } SVgroupChangeInfo;

 static int32_t         mndNodeCheckSentinel = 0;
-static SStreamExecNodeInfo     execNodeList;
+static SStreamExecInfo execInfo;

 static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
@ -77,19 +77,21 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
                                                  int64_t streamId, int32_t taskId);
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
 static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
-
 static SArray *extractNodeListFromStream(SMnode *pMnode);
-static SArray           *mndTakeVgroupSnapshot(SMnode *pMnode);
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady);
+
 static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);

 static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name);
 static int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans);
-static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset);
-static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo);
-
-static void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode);
-static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode);
+static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
+                            int32_t retryCode);
+static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
+static void    removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
+static void    keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
 static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot);
+static int32_t doKillActiveCheckpointTrans(SMnode *pMnode);
+static int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList);

 int32_t mndInitStream(SMnode *pMnode) {
  SSdbTable table = {
@ -129,18 +131,18 @@ int32_t mndInitStream(SMnode *pMnode) {
  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);

-  taosThreadMutexInit(&execNodeList.lock, NULL);
-  execNodeList.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
-  execNodeList.pTaskList = taosArrayInit(4, sizeof(STaskId));
+  taosThreadMutexInit(&execInfo.lock, NULL);
+  execInfo.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
+  execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));

  return sdbSetTable(pMnode->pSdb, table);
 }

 void mndCleanupStream(SMnode *pMnode) {
-  taosArrayDestroy(execNodeList.pTaskList);
-  taosHashCleanup(execNodeList.pTaskMap);
-  taosThreadMutexDestroy(&execNodeList.lock);
-  mDebug("mnd stream cleanup");
+  taosArrayDestroy(execInfo.pTaskList);
+  taosHashCleanup(execInfo.pTaskMap);
+  taosThreadMutexDestroy(&execInfo.lock);
+  mDebug("mnd stream exec info cleanup");
 }

 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@ -286,7 +288,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
 static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
  int8_t status = atomic_load_8(&pStream->status);
  if (status == STREAM_STATUS__NORMAL) {
-    strcpy(dst, "normal");
+    strcpy(dst, "ready");
  } else if (status == STREAM_STATUS__STOP) {
    strcpy(dst, "stop");
  } else if (status == STREAM_STATUS__FAILED) {
@ -294,7 +296,7 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
  } else if (status == STREAM_STATUS__RECOVER) {
    strcpy(dst, "recover");
  } else if (status == STREAM_STATUS__PAUSE) {
-    strcpy(dst, "pause");
+    strcpy(dst, "paused");
  }
 }

@ -516,7 +518,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {

  STransAction action = {0};
  action.mTraceId = pTrans->mTraceId;
-  initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet);
+  initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0);
  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(buf);
    return -1;
@ -688,7 +690,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
  pReq->streamId = pTask->id.streamId;

  STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet, 0);
  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
@ -847,10 +849,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {

  mndTransDrop(pTrans);

-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
  mDebug("register to stream task node list");
-  keepStreamTasksInBuf(&streamObj, &execNodeList);
-  taosThreadMutexUnlock(&execNodeList.lock);
+  keepStreamTasksInBuf(&streamObj, &execInfo);
+  taosThreadMutexUnlock(&execInfo.lock);

  code = TSDB_CODE_ACTION_IN_PROGRESS;

@ -882,9 +884,8 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
    return 0;
  }

-  int64_t                  checkpointId = taosGetTimestampMs();
  SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
-  pMsg->checkpointId = checkpointId;
+  pMsg->checkpointId = taosGetTimestampMs();

  int32_t size = sizeof(SMStreamDoCheckpointMsg);
  SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@ -936,7 +937,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
 }
 // static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) {
 //   int64_t timestampMs = taosGetTimestampMs();
-//   if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointTickInterval * 1000) {
+//   if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000) {
 //     return -1;
 //   }

@ -1069,7 +1070,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream

        STransAction action = {0};
        SEpSet       epset = mndGetVgroupEpset(pMnode, pVgObj);
-        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset);
+        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
        mndReleaseVgroup(pMnode, pVgObj);

        if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@ -1084,6 +1085,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
  pStream->checkpointId = chkptId;
  pStream->checkpointFreq = taosGetTimestampMs();
  pStream->currentTick = 0;
+
  // 3. commit log: stream checkpoint info
  pStream->version = pStream->version + 1;

@ -1133,31 +1135,37 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
  {  // check if the node update happens or not
    int64_t ts = taosGetTimestampSec();

-    if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) {
-      if (execNodeList.pNodeEntryList != NULL) {
-        execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+    if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
+      if (execInfo.pNodeEntryList != NULL) {
+        execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
      }

-      execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode);
+      execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
    }

-    if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) {
+    if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
      mDebug("stream task node change checking done, no vgroups exist, do nothing");
-      execNodeList.ts = ts;
+      execInfo.ts = ts;
      return 0;
    }

-    for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) {
-      SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, i);
+    for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
+      SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
      if (pNodeEntry->stageUpdated) {
        mDebug("stream task not ready due to node update detected, checkpoint not issued");
        return 0;
      }
    }

-    SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode);
+    bool allReady = true;
+    SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
+    if (!allReady) {
+      mWarn("not all vnodes are ready, ignore the checkpoint")
+      taosArrayDestroy(pNodeSnapshot);
+      return 0;
+    }

-    SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot);
+    SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
    bool              nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0);
    taosArrayDestroy(changeInfo.pUpdateNodeList);
    taosHashCleanup(changeInfo.pDBMap);
@ -1169,26 +1177,25 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
    }
  }

-  {  // check if all tasks are in TASK_STATUS__NORMAL status
+  {  // check if all tasks are in TASK_STATUS__READY status
    bool ready = true;

-    taosThreadMutexLock(&execNodeList.lock);
-    for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pTaskList); ++i) {
-      STaskId *p = taosArrayGet(execNodeList.pTaskList, i);
-      STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, p, sizeof(*p));
+    taosThreadMutexLock(&execInfo.lock);
+    for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
+      STaskId *p = taosArrayGet(execInfo.pTaskList, i);
+      STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
      if (pEntry == NULL) {
        continue;
      }

-      if (pEntry->status != TASK_STATUS__NORMAL) {
+      if (pEntry->status != TASK_STATUS__READY) {
        mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued",
-              pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamGetTaskStatusStr(pEntry->status));
+              pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status));
        ready = false;
        break;
      }
    }
-    taosThreadMutexUnlock(&execNodeList.lock);
-
+    taosThreadMutexUnlock(&execInfo.lock);
    if (!ready) {
      return 0;
    }
@ -1202,7 +1209,8 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
    mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
    return -1;
  }
-  mDebug("start to trigger checkpoint, checkpointId: %" PRId64 "", checkpointId);
+
+  mDebug("start to trigger checkpoint, checkpointId: %" PRId64, checkpointId);

  const char *pDb = mndGetStreamDB(pMnode);
  mndTransSetDbName(pTrans, pDb, "checkpoint");
@ -1228,11 +1236,16 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {

  if (code == 0) {
    if (mndTransPrepare(pMnode, pTrans) != 0) {
-      mError("failed to prepre trans rebalance since %s", terrstr());
+      mError("failed to prepare trans rebalance since %s", terrstr());
    }
  }

  mndTransDrop(pTrans);
+
+  // only one trans here
+  taosThreadMutexLock(&execInfo.lock);
+  execInfo.activeCheckpoint = checkpointId;
+  taosThreadMutexUnlock(&execInfo.lock);
  return code;
 }

@ -1310,7 +1323,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
    return -1;
  }

-  removeStreamTasksInBuf(pStream, &execNodeList);
+  removeStreamTasksInBuf(pStream, &execInfo);

  SName name = {0};
  tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB);
@ -1561,12 +1574,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
        char status[20 + VARSTR_HEADER_SIZE] = {0};

        STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-        STaskStatusEntry* pe = taosHashGet(execNodeList.pTaskMap, &id, sizeof(id));
+        STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
        if (pe == NULL) {
          continue;
        }

-        const char* pStatus = streamGetTaskStatusStr(pe->status);
+        const char* pStatus = streamTaskGetStatusStr(pe->status);
        STR_TO_VARSTR(status, pStatus);

        // status
@ -1641,7 +1654,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) {
  pReq->streamId = pTask->id.streamId;

  STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet, 0);
  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
@ -1774,7 +1787,7 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t ig
  pReq->igUntreated = igUntreated;

  STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet, 0);
  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
@ -1878,18 +1891,19 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
  return TSDB_CODE_ACTION_IN_PROGRESS;
 }

-static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, int64_t streamId,
-                              int32_t taskId) {
-  pMsg->streamId = streamId;
-  pMsg->taskId = taskId;
+static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
+                              int32_t transId) {
+  pMsg->streamId = pId->streamId;
+  pMsg->taskId = pId->taskId;
+  pMsg->transId = transId;
  pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
  taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
 }

 static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
-                                          int64_t streamId, int32_t taskId) {
+                                          SStreamTaskId* pId, int32_t transId) {
  SStreamTaskNodeUpdateMsg req = {0};
-  initNodeUpdateMsg(&req, pInfo, streamId, taskId);
+  initNodeUpdateMsg(&req, pInfo, pId, transId);

  int32_t code = 0;
  int32_t blen;
@ -1953,20 +1967,19 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans) {
  return 0;
 }

-void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset) {
+void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
+                     int32_t retryCode) {
  pAction->epSet = *pEpset;
  pAction->contLen = contLen;
  pAction->pCont = pCont;
  pAction->msgType = msgType;
+  pAction->retryCode = retryCode;
 }

 // todo extract method: traverse stream tasks
 // build trans to update the epset
-static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo) {
-  STrans* pTrans = doCreateTrans(pMnode, pStream, "stream-task-update");
-  if (pTrans == NULL) {
-    return terrno;
-  }
+static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
+  mDebug("start to build stream:0x%" PRIx64 " tasks epset update", pStream->uid);

  taosWLockLatch(&pStream->lock);
  int32_t numOfLevels = taosArrayGetSize(pStream->tasks);
@ -1981,38 +1994,20 @@ static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgr
      void   *pBuf = NULL;
      int32_t len = 0;
      streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
-      doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId);
+      doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);

      STransAction action = {0};
-      initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet);
+      initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
      if (mndTransAppendRedoAction(pTrans, &action) != 0) {
        taosMemoryFree(pBuf);
        taosWUnLockLatch(&pStream->lock);
-        mndTransDrop(pTrans);
        return -1;
      }
    }
  }

  taosWUnLockLatch(&pStream->lock);
-
-  int32_t code = mndPersistTransLog(pStream, pTrans);
-  if (code != TSDB_CODE_SUCCESS) {
-    sdbRelease(pMnode->pSdb, pStream);
-    return -1;
-  }
-
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
-    sdbRelease(pMnode->pSdb, pStream);
-    mndTransDrop(pTrans);
-    return -1;
-  }
-
-  sdbRelease(pMnode->pSdb, pStream);
-  mndTransDrop(pTrans);
-
-  return TSDB_CODE_ACTION_IN_PROGRESS;
+  return 0;
 }

 static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) {
@ -2050,8 +2045,8 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP

          char buf[256] = {0};
          EPSET_TO_STR(&pCurrent->epset, buf);
-          mDebug("nodeId:%d epset changed detected, old:%s:%d -> new:%s", pCurrent->nodeId, pPrevEp->fqdn,
-                 pPrevEp->port, buf);
+          mDebug("nodeId:%d restart/epset changed detected, old:%s:%d -> new:%s, stageUpdate:%d", pCurrent->nodeId,
+                 pPrevEp->fqdn, pPrevEp->port, buf, pPrevEntry->stageUpdated);

          SNodeUpdateInfo updateInfo = {.nodeId = pPrevEntry->nodeId};
          epsetAssign(&updateInfo.prevEp, &pPrevEntry->epset);
@ -2071,11 +2066,12 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
  return info;
 }

-static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) {
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) {
  SSdb   *pSdb = pMnode->pSdb;
  void   *pIter = NULL;
  SVgObj *pVgroup = NULL;

+  *allReady = true;
  SArray *pVgroupListSnapshot = taosArrayInit(4, sizeof(SNodeEntry));

  while (1) {
@ -2087,8 +2083,26 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) {
    SNodeEntry entry = {0};
    entry.epset = mndGetVgroupEpset(pMnode, pVgroup);
    entry.nodeId = pVgroup->vgId;
-    entry.hbTimestamp = -1;
+    entry.hbTimestamp = pVgroup->updateTime;

+    if (*allReady) {
+      for (int32_t i = 0; i < pVgroup->replica; ++i) {
+        if (!pVgroup->vnodeGid[i].syncRestore) {
+          *allReady = false;
+          break;
+        }
+
+        ESyncState state = pVgroup->vnodeGid[i].syncState;
+        if (state == TAOS_SYNC_STATE_OFFLINE || state == TAOS_SYNC_STATE_ERROR) {
+          *allReady = false;
+          break;
+        }
+      }
+    }
+
+    char buf[256] = {0};
+    EPSET_TO_STR(&entry.epset, buf);
+    mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
    taosArrayPush(pVgroupListSnapshot, &entry);
    sdbRelease(pSdb, pVgroup);
  }
@ -2102,27 +2116,63 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
  // check all streams that involved this vnode should update the epset info
  SStreamObj *pStream = NULL;
  void       *pIter = NULL;
+  STrans     *pTrans = NULL;
+
  while (1) {
    pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
    if (pIter == NULL) {
      break;
    }

+    // here create only one trans
+    if (pTrans == NULL) {
+      pTrans = doCreateTrans(pMnode, pStream, "stream-task-update");
+      if (pTrans == NULL) {
+        sdbRelease(pSdb, pStream);
+        sdbCancelFetch(pSdb, pIter);
+        return terrno;
+      }
+    }
+
    void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb));
    void *p1 = taosHashGet(pChangeInfo->pDBMap, pStream->sourceDb, strlen(pStream->sourceDb));
    if (p == NULL && p1 == NULL) {
-      mndReleaseStream(pMnode, pStream);
+      mDebug("stream:0x%" PRIx64 " %s not involved nodeUpdate, ignore", pStream->uid, pStream->name);
+      sdbRelease(pSdb, pStream);
      continue;
    }

-    mDebug("stream:0x%" PRIx64 " involved node changed, create update trans", pStream->uid);
-    int32_t code = createStreamUpdateTrans(pMnode, pStream, pChangeInfo);
+    mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
+           pStream->name, pTrans->id);
+
+    int32_t code = createStreamUpdateTrans(pStream, pChangeInfo, pTrans);
+
+    // todo: not continue, drop all and retry again
+    if (code != TSDB_CODE_SUCCESS) {
+      mError("stream:0x%" PRIx64 " build nodeUpdate trans failed, ignore and continue, code:%s", pStream->uid,
+             tstrerror(code));
+      sdbRelease(pSdb, pStream);
+      continue;
+    }
+
+    code = mndPersistTransLog(pStream, pTrans);
+    sdbRelease(pSdb, pStream);
+
    if (code != TSDB_CODE_SUCCESS) {
      sdbCancelFetch(pSdb, pIter);
-      return code;
+      return -1;
    }
  }

+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
+    sdbRelease(pMnode->pSdb, pStream);
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  sdbRelease(pMnode->pSdb, pStream);
+  mndTransDrop(pTrans);
  return 0;
 }

@ -2165,6 +2215,10 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
  while ((pIter = taosHashIterate(pHash, pIter)) != NULL) {
    SNodeEntry *pEntry = (SNodeEntry *)pIter;
    taosArrayPush(plist, pEntry);
+
+    char buf[256] = {0};
+    EPSET_TO_STR(&pEntry->epset, buf);
+    mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf);
  }
  taosHashCleanup(pHash);

@ -2182,28 +2236,31 @@ static void doExtractTasksFromStream(SMnode *pMnode) {
      break;
    }

-    keepStreamTasksInBuf(pStream, &execNodeList);
+    keepStreamTasksInBuf(pStream, &execInfo);
    sdbRelease(pSdb, pStream);
  }
 }

-static int32_t doRemoveFromTask(SStreamExecNodeInfo* pExecNode, STaskId* pRemovedId) {
+static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
  void *p = taosHashGet(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
+  if (p == NULL) {
+    return TSDB_CODE_SUCCESS;
+  }

-  if (p != NULL) {
  taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));

-    for(int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
-      STaskId* pId = taosArrayGet(pExecNode->pTaskList, k);
+  for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
+    STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
    if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) {
      taosArrayRemove(pExecNode->pTaskList, k);
-        mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t) pRemovedId->taskId,
-              (int32_t)taosArrayGetSize(pExecNode->pTaskList));
+
+      int32_t num = taosArrayGetSize(pExecNode->pTaskList);
+      mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)pRemovedId->taskId, num);
      break;
    }
  }
-  }
-  return 0;
+
+  return TSDB_CODE_SUCCESS;
 }

 static bool taskNodeExists(SArray* pList, int32_t nodeId) {
@ -2220,31 +2277,31 @@ static bool taskNodeExists(SArray* pList, int32_t nodeId) {
 }

 int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
-  SArray* pRemoveTaskList = taosArrayInit(4, sizeof(STaskId));
+  SArray* pRemovedTasks = taosArrayInit(4, sizeof(STaskId));

-  int32_t numOfTask = taosArrayGetSize(execNodeList.pTaskList);
+  int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
  for(int32_t i = 0; i < numOfTask; ++i) {
-    STaskId* pId = taosArrayGet(execNodeList.pTaskList, i);
-    STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, pId, sizeof(*pId));
+    STaskId* pId = taosArrayGet(execInfo.pTaskList, i);
+    STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));

    bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
    if (!existed) {
-      taosArrayPush(pRemoveTaskList, pId);
+      taosArrayPush(pRemovedTasks, pId);
    }
  }

-  for(int32_t i = 0; i < taosArrayGetSize(pRemoveTaskList); ++i) {
-    STaskId* pId = taosArrayGet(pRemoveTaskList, i);
-    doRemoveFromTask(&execNodeList, pId);
+  for(int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) {
+    STaskId* pId = taosArrayGet(pRemovedTasks, i);
+    doRemoveTasks(&execInfo, pId);
  }

-  mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemoveTaskList),
-         (int32_t) taosArrayGetSize(execNodeList.pTaskList));
+  mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
+         (int32_t) taosArrayGetSize(execInfo.pTaskList));

  int32_t size = taosArrayGetSize(pNodeSnapshot);
  SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry));
-  for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) {
-    SNodeEntry* p = taosArrayGet(execNodeList.pNodeEntryList, i);
+  for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
+    SNodeEntry* p = taosArrayGet(execInfo.pNodeEntryList, i);

    for(int32_t j = 0; j < size; ++j) {
      SNodeEntry* pEntry = taosArrayGet(pNodeSnapshot, j);
@ -2255,10 +2312,11 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
    }
  }

-  execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
-  execNodeList.pNodeEntryList = pValidNodeEntryList;
+  execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
+  execInfo.pNodeEntryList = pValidNodeEntryList;

-  taosArrayDestroy(pRemoveTaskList);
+  mDebug("remain %d valid node entries", (int32_t) taosArrayGetSize(pValidNodeEntryList));
+  taosArrayDestroy(pRemovedTasks);
  return 0;
 }

@ -2275,43 +2333,55 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
  int64_t ts = taosGetTimestampSec();

  SMnode *pMnode = pMsg->info.node;
-  if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) {
-    if (execNodeList.pNodeEntryList != NULL) {
-      execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+  if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
+    if (execInfo.pNodeEntryList != NULL) {
+      execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
+    }
+    execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
  }

-    execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode);
-  }
-
-  if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) {
+  if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
    mDebug("end to do stream task node change checking, no vgroup exists, do nothing");
-    execNodeList.ts = ts;
+    execInfo.ts = ts;
    atomic_store_32(&mndNodeCheckSentinel, 0);
    return 0;
  }

-  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode);
+  bool allVnodeReady = true;
+  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVnodeReady);
+  if (!allVnodeReady) {
+    taosArrayDestroy(pNodeSnapshot);
+    atomic_store_32(&mndNodeCheckSentinel, 0);
+    mWarn("not all vnodes are ready, ignore the exec nodeUpdate check");
+    return 0;
+  }

-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
  removeExpirednodeEntryAndTask(pNodeSnapshot);

-  SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot);
+  SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
  if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) {
+
+    // kill current active checkpoint transaction, since the transaction is vnode wide.
+    doKillActiveCheckpointTrans(pMnode);
    code = mndProcessVgroupChange(pMnode, &changeInfo);

    // keep the new vnode snapshot
    if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) {
      mDebug("create trans successfully, update cached node list");
-      taosArrayDestroy(execNodeList.pNodeEntryList);
-      execNodeList.pNodeEntryList = pNodeSnapshot;
-      execNodeList.ts = ts;
+      taosArrayDestroy(execInfo.pNodeEntryList);
+      execInfo.pNodeEntryList = pNodeSnapshot;
+      execInfo.ts = ts;
+    } else {
+      mDebug("unexpect code during create nodeUpdate trans, code:%s", tstrerror(code));
+      taosArrayDestroy(pNodeSnapshot);
    }
  } else {
    mDebug("no update found in nodeList");
    taosArrayDestroy(pNodeSnapshot);
  }

-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);
  taosArrayDestroy(changeInfo.pUpdateNodeList);
  taosHashCleanup(changeInfo.pDBMap);

@ -2324,10 +2394,6 @@ typedef struct SMStreamNodeCheckMsg {
  int8_t placeHolder;  // // to fix windows compile error, define place holder
 } SMStreamNodeCheckMsg;

-typedef struct SMStreamTaskResetMsg {
-  int8_t placeHolder;
-} SMStreamTaskResetMsg;
-
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
  SMnode *pMnode = pReq->info.node;
  SSdb   *pSdb = pMnode->pSdb;
@ -2343,7 +2409,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
  return 0;
 }

-void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) {
+void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
  int32_t level = taosArrayGetSize(pStream->tasks);

  for (int32_t i = 0; i < level; i++) {
@ -2368,7 +2434,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) {
  }
 }

-void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode) {
+void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
  int32_t level = taosArrayGetSize(pStream->tasks);
  for (int32_t i = 0; i < level; i++) {
    SArray *pLevel = taosArrayGetP(pStream->tasks, i);
@ -2399,7 +2465,7 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode
  ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
 }

-static STrans* doCreateTrans(SMnode* pMnode, SStreamObj* pStream, const char* name) {
+STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name) {
  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, name);
  if (pTrans == NULL) {
    mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
@ -2451,7 +2517,7 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
      pReq->streamId = pTask->id.streamId;

      STransAction action = {0};
-      initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet);
+      initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet, 0);
      if (mndTransAppendRedoAction(pTrans, &action) != 0) {
        taosMemoryFree(pReq);
        taosWUnLockLatch(&pStream->lock);
@ -2482,14 +2548,12 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
  return TSDB_CODE_ACTION_IN_PROGRESS;
 }

-int32_t mndResetFromCheckpoint(SMnode* pMnode) {
-  // find the checkpoint trans id
+int32_t doKillActiveCheckpointTrans(SMnode *pMnode) {
  int32_t transId = 0;
-
-  {
  SSdb   *pSdb = pMnode->pSdb;
  STrans *pTrans = NULL;
-    void* pIter = NULL;
+  void   *pIter = NULL;
+
  while (1) {
    pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans);
    if (pIter == NULL) {
@ -2505,15 +2569,22 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {

    sdbRelease(pSdb, pTrans);
  }
-  }

  if (transId == 0) {
    mError("failed to find the checkpoint trans, reset not executed");
    return TSDB_CODE_SUCCESS;
  }

-  STrans* pTrans = mndAcquireTrans(pMnode, transId);
+  pTrans = mndAcquireTrans(pMnode, transId);
+  mInfo("kill checkpoint trans:%d", transId);
+
  mndKillTrans(pMnode, pTrans);
+  mndReleaseTrans(pMnode, pTrans);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mndResetFromCheckpoint(SMnode* pMnode) {
+  doKillActiveCheckpointTrans(pMnode);

  // set all tasks status to be normal, refactor later to be stream level, instead of vnode level.
  SSdb *pSdb = pMnode->pSdb;
@ -2525,6 +2596,7 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
      break;
    }

+    // todo this transaction should exist be only one
    mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid);
    int32_t code = createStreamResetStatusTrans(pMnode, pStream);
    if (code != TSDB_CODE_SUCCESS) {
@ -2536,6 +2608,43 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
  return 0;
 }

+int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) {
+  int32_t num = taosArrayGetSize(pNodeList);
+
+  for (int k = 0; k < num; ++k) {
+    int32_t* pVgId = taosArrayGet(pNodeList, k);
+
+    int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
+    for (int i = 0; i < numOfNodes; ++i) {
+      SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
+
+      if (pNodeEntry->nodeId == *pVgId) {
+        mInfo("vgId:%d expired in stream task, needs update nodeEp", *pVgId);
+        pNodeEntry->stageUpdated = true;
+        break;
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) {
+  int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
+  for(int32_t j = 0; j < numOfNodes; ++j) {
+    SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j);
+    if (pNodeEntry->nodeId == pTaskEntry->nodeId) {
+
+      mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId,
+            pTaskEntry->stage, stage, pTaskEntry->id.taskId);
+
+      pNodeEntry->stageUpdated = true;
+      pTaskEntry->stage = stage;
+      break;
+    }
+  }
+}
+
 int32_t mndProcessStreamHb(SRpcMsg *pReq) {
  SMnode      *pMnode = pReq->info.node;
  SStreamHbMsg req = {0};
@ -2555,35 +2664,26 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {

  mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks);

-  taosThreadMutexLock(&execNodeList.lock);
-  int32_t numOfExisted = taosHashGetSize(execNodeList.pTaskMap);
+  taosThreadMutexLock(&execInfo.lock);
+  int32_t numOfExisted = taosHashGetSize(execInfo.pTaskMap);
  if (numOfExisted == 0) {
    doExtractTasksFromStream(pMnode);
  }

+  setNodeEpsetExpiredFlag(req.pUpdateNodes);
+
  for (int32_t i = 0; i < req.numOfTasks; ++i) {
    STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i);
-    STaskStatusEntry *pEntry = taosHashGet(execNodeList.pTaskMap, &p->id, sizeof(p->id));
-    if (pEntry == NULL) {
+    STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id));
+    if (pTaskEntry == NULL) {
      mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId);
      continue;
    }

-    if (p->stage != pEntry->stage && pEntry->stage != -1) {
-      int32_t numOfNodes = taosArrayGetSize(execNodeList.pNodeEntryList);
-      for(int32_t j = 0; j < numOfNodes; ++j) {
-        SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, j);
-        if (pNodeEntry->nodeId == pEntry->nodeId) {
-          mInfo("vgId:%d stage updated, from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64,
-                pEntry->nodeId, pEntry->stage, p->stage, pEntry->id.taskId);
-
-          pNodeEntry->stageUpdated = true;
-          pEntry->stage = p->stage;
-          break;
-        }
-      }
+    if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) {
+      updateStageInfo(pTaskEntry, p->stage);
    } else {
-      streamTaskStatusCopy(pEntry, p);
+      streamTaskStatusCopy(pTaskEntry, p);
      if (p->activeCheckpointId != 0) {
        if (activeCheckpointId != 0) {
          ASSERT(activeCheckpointId == p->activeCheckpointId);
@ -2597,26 +2697,32 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
      }
    }

-    pEntry->status = p->status;
-    if (p->status != TASK_STATUS__NORMAL) {
-      mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamGetTaskStatusStr(p->status));
+    pTaskEntry->status = p->status;
+    if (p->status != TASK_STATUS__READY) {
+      mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status));
    }
  }

  // current checkpoint is failed, rollback from the checkpoint trans
  // kill the checkpoint trans and then set all tasks status to be normal
  if (checkpointFailed && activeCheckpointId != 0) {
-    if (execNodeList.activeCheckpoint != activeCheckpointId) {
-      mInfo("checkpointId:%"PRId64" failed, issue task-reset trans to reset all tasks status", activeCheckpointId);
-      execNodeList.activeCheckpoint = activeCheckpointId;
+    bool allReady = true;
+    SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady);
+    taosArrayDestroy(p);
+
+    if (allReady) {
+      // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
+      mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status",
+            execInfo.activeCheckpoint);
      mndResetFromCheckpoint(pMnode);
    } else {
-      mDebug("checkpoint:%"PRId64" reset has issued already, ignore it", activeCheckpointId);
+      mInfo("not all vgroups are ready, wait for next HB from stream tasks");
    }
  }

-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);

  taosArrayDestroy(req.pTaskStatus);
+  taosArrayDestroy(req.pUpdateNodes);
  return TSDB_CODE_SUCCESS;
 }
--- a/source/dnode/mnode/impl/src/mndSync.c
+++ b/source/dnode/mnode/impl/src/mndSync.c
@ -95,7 +95,11 @@ static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STr
  }

 _OUT:
+  if (pRow) {
+    SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type];
+    if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false);
    taosMemoryFreeClear(pRow);
+  }
  return code;
 }

--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@ -754,10 +754,14 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup
  for (int32_t v = 0; v < pVgroup->replica; ++v) {
    SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
    SDnodeObj *pDnode = taosArrayGet(pArray, v);
-    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+    if (pDnode == NULL) {
      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
      return -1;
    }
+    if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
+      return -1;
+    }

    int64_t vgMem = mndGetVgroupMemory(pMnode, pDb, pVgroup);
    if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@ -1180,10 +1184,14 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro
    }
    if (used) continue;

-    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+    if (pDnode == NULL) {
      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
      return -1;
    }
+    if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
+      return -1;
+    }

    int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
    if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@ -1912,7 +1920,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
    if (numOfVnodes >= pNew1->numOfSupportVnodes) {
      mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes,
             pNew1->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
      goto _OVER;
    }

@ -1935,7 +1943,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
    if (numOfVnodes >= pNew2->numOfSupportVnodes) {
      mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes,
             pNew2->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
      goto _OVER;
    }
    int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
@ -1956,7 +1964,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
    if (numOfVnodes >= pNew3->numOfSupportVnodes) {
      mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes,
             pNew3->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
      goto _OVER;
    }
    int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@ -92,11 +92,13 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
    }
  }

-  qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64
-         " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms",
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
+  qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
+        " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms",
        SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-         pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
-         pTask->info.fillHistory, pTask->info.triggerParam);
+        pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.triggerParam);

  return 0;
 }
@ -174,9 +176,15 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {

  int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta);
  taosWUnLockLatch(&pSnode->pMeta->lock);
-  qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr,
-         streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks);

+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
+  qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE,
+         pTask->id.idStr, p, numOfTasks);
+
+  EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+  streamTaskHandleEvent(pTask->status.pSM, event);
  streamTaskCheckDownstream(pTask);
  return 0;
 }
@ -352,10 +360,10 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) {
  if (pTask != NULL) {
    rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
    streamMetaReleaseTask(pSnode->pMeta, pTask);
-
-    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+    char* p = NULL;
+    streamTaskGetStatus(pTask, &p);
    qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d",
-            pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+            pTask->id.idStr, p, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
  } else {
    rsp.status = TASK_DOWNSTREAM_NOT_READY;
    qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d",
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@ -126,7 +126,6 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
 int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
 int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
                      int32_t type, int32_t vgId);
-//int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId);
 int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);

 // tqMeta
@ -134,7 +133,6 @@ int32_t tqMetaOpen(STQ* pTq);
 int32_t tqMetaClose(STQ* pTq);
 int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
 int32_t tqMetaDeleteHandle(STQ* pTq, const char* key);
-//int32_t tqMetaRestoreHandle(STQ* pTq);
 int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen);
 int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
 int32_t tqMetaRestoreCheckInfo(STQ* pTq);
@ -160,8 +158,8 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
 // tqStream
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
 int32_t tqScanWal(STQ* pTq);
-int32_t tqCheckAndRunStreamTask(STQ* pTq);
-int32_t tqStartStreamTasks(STQ* pTq);
+int32_t tqStartStreamTask(STQ* pTq);
+int32_t tqResetStreamTaskStatus(STQ* pTq);
 int32_t tqStopStreamTasks(STQ* pTq);

 // tq util
@ -171,6 +169,12 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
                        int32_t type, int64_t sver, int64_t ever);
 int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset);
 void    tqUpdateNodeStage(STQ* pTq, bool isLeader);
+int32_t setDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
+                               SSubmitTbData* pTableData, const char* id);
+int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
+
+SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
+                                       SSDataBlock* pDataBlock, SArray* pTagArray);

 #ifdef __cplusplus
 }
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@ -309,7 +309,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STs
 void    tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive);

 // tsdbMerge.c ==============================================================================================
-int32_t tsdbMerge(void *arg);
+int32_t tsdbSchedMerge(STsdb *tsdb, int32_t fid);

 // tsdbDiskData ==============================================================================================
 int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
@ -371,7 +371,7 @@ struct STsdb {
  char                *path;
  SVnode              *pVnode;
  STsdbKeepCfg         keepCfg;
-  TdThreadRwlock       rwLock;
+  TdThreadMutex        mutex;
  SMemTable           *mem;
  SMemTable           *imem;
  STsdbFS              fs;  // old
@ -670,7 +670,7 @@ struct SDelFWriter {
 };

 #include "tarray2.h"
-//#include "tsdbFS2.h"
+// #include "tsdbFS2.h"
 //  struct STFileSet;
 typedef struct STFileSet STFileSet;
 typedef TARRAY2(STFileSet *) TFileSetArray;
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@ -231,7 +231,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
-int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq);
+int32_t tqLaunchStreamTaskAsync(STQ* pTq);

 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
--- a/source/dnode/vnode/src/sma/smaOpen.c
+++ b/source/dnode/vnode/src/sma/smaOpen.c
@ -33,7 +33,7 @@ static int32_t rsmaRestore(SSma *pSma);
 #define SMA_OPEN_RSMA_IMPL(v, l, force)                                                             \
  do {                                                                                              \
    SRetention *r = (SRetention *)VND_RETENTIONS(v) + l;                                            \
-    if (!RETENTION_VALID(r)) {                                                               \
+    if (!RETENTION_VALID(l, r)) {                                                                   \
      if (l == 0) {                                                                                 \
        code = TSDB_CODE_INVALID_PARA;                                                              \
        TSDB_CHECK_CODE(code, lino, _exit);                                                         \
@ -79,20 +79,18 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
  freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE);
  keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE);

-  int32_t nFreqTimes = (r + level)->freq / (r + TSDB_RETENTION_L0)->freq;
+  int32_t nFreqTimes = (r + level)->freq / (60 * 1000);  // use 60s for freq of 1st level
  days *= (nFreqTimes > 1 ? nFreqTimes : 1);

-  if (days > keepDuration) {
-    days = keepDuration;
-  }
-
-  if (days > TSDB_MAX_DURATION_PER_FILE) {
-    days = TSDB_MAX_DURATION_PER_FILE;
-  }
-
  if (days < freqDuration) {
    days = freqDuration;
  }
+
+  int32_t maxKeepDuration = TMIN(keepDuration, TSDB_MAX_DURATION_PER_FILE);
+  if (days > maxKeepDuration) {
+    days = maxKeepDuration;
+  }
+
 _exit:
  smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration);
  return days;
@ -157,6 +155,7 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) {
 _exit:
  if (code) {
    smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
+    terrno = code;
  }
  return code;
 }
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@ -31,8 +31,6 @@ SSmaMgmt smaMgmt = {

 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;

-extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now);
-
 static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
 static void       tdUidStoreDestory(STbUidStore *pStore);
 static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd);
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@ -35,8 +35,8 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) {
  return code;
 }

-int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) {
-  int32_t code = tdProcessTSmaCreateImpl(pSma, version, msg);
+int32_t tdProcessTSmaCreate(SSma *pSma, int64_t ver, const char *msg) {
+  int32_t code = tdProcessTSmaCreateImpl(pSma, ver, msg);

  return code;
 }
@ -109,7 +109,7 @@ _exit:
 * @param pMsg
 * @return int32_t
 */
-static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) {
+static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t ver, const char *pMsg) {
  int32_t        code = 0;
  int32_t        lino = 0;
  SSmaCfg       *pCfg = (SSmaCfg *)pMsg;
@ -118,7 +118,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *

  if (TD_VID(pSma->pVnode) == pCfg->dstVgId) {
    // create tsma meta in dstVgId
-    if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) {
+    if (metaCreateTSma(SMA_META(pSma), ver, pCfg) < 0) {
      code = terrno;
      TSDB_CHECK_CODE(code, lino, _exit);
    }
@ -130,7 +130,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *
    pReq.schemaRow = pCfg->schemaRow;
    pReq.schemaTag = pCfg->schemaTag;

-    if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) {
+    if (metaCreateSTable(SMA_META(pSma), ver, &pReq) < 0) {
      code = terrno;
      TSDB_CHECK_CODE(code, lino, _exit);
    }
@ -154,94 +154,36 @@ _exit:
  return code;
 }

-int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema,
-                         SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName,
-                         SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
+int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, int64_t suid,
+                         const char *stbFullName, SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
  int32_t      code = 0;
  int32_t      lino = 0;
  void        *pBuf = NULL;
  int32_t      len = 0;
  SSubmitReq2 *pReq = NULL;
  SArray      *tagArray = NULL;
-  SArray      *createTbArray = NULL;
-  SArray      *pVals = NULL;

-  int32_t sz = taosArrayGetSize(pBlocks);
+  int32_t numOfBlocks = taosArrayGetSize(pBlocks);

  tagArray = taosArrayInit(1, sizeof(STagVal));
-  createTbArray = taosArrayInit(sz, POINTER_BYTES);
  pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2));
-  pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));

-  if(!tagArray || !createTbArray || !pReq || !pReq->aSubmitTbData) {
+  if (!tagArray || !pReq) {
    code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
    TSDB_CHECK_CODE(code, lino, _exit);
  }

-  // create table req
-  if (createTb) {
-    for (int32_t i = 0; i < sz; ++i) {
-      SSDataBlock   *pDataBlock = taosArrayGet(pBlocks, i);
-      SVCreateTbReq *pCreateTbReq = NULL;
-      if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
-        taosArrayPush(createTbArray, &pCreateTbReq);
-        continue;
-      }
-
-      if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        TSDB_CHECK_CODE(code, lino, _exit);
-      };
-
-      // don't move to the end of loop as to destroy in the end of func when error occur
-      taosArrayPush(createTbArray, &pCreateTbReq);
-
-      // set const
-      pCreateTbReq->flags = 0;
-      pCreateTbReq->type = TSDB_CHILD_TABLE;
-      pCreateTbReq->ctb.suid = suid;
-
-      // set super table name
-      SName name = {0};
-      tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-      pCreateTbReq->ctb.stbName = taosStrdup((char *)tNameGetTableName(&name));  // taosStrdup(stbFullName);
-
-      // set tag content
-      taosArrayClear(tagArray);
-      STagVal tagVal = {
-          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
-          .type = TSDB_DATA_TYPE_UBIGINT,
-          .i64 = (int64_t)pDataBlock->info.id.groupId,
-      };
-      taosArrayPush(tagArray, &tagVal);
-      pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
-
-      STag *pTag = NULL;
-      tTagNew(tagArray, 1, false, &pTag);
-      if (pTag == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
+  pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));
+  if (pReq->aSubmitTbData == NULL) {
+    code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
    TSDB_CHECK_CODE(code, lino, _exit);
  }
-      pCreateTbReq->ctb.pTag = (uint8_t *)pTag;

-      // set tag name
-      SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-      char    tagNameStr[TSDB_COL_NAME_LEN] = {0};
-      strcpy(tagNameStr, "group_id");
-      taosArrayPush(tagName, tagNameStr);
-      pCreateTbReq->ctb.tagName = tagName;
-
-      // set table name
-      if (pDataBlock->info.parTbName[0]) {
-        pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
-      } else {
-        pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId);
-      }
-    }
-  }
+  SHashObj *pTableIndexMap =
+      taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);

  // SSubmitTbData req
-  for (int32_t i = 0; i < sz; ++i) {
+  for (int32_t i = 0; i < numOfBlocks; ++i) {
    SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i);
    if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
      pDeleteReq->suid = suid;
@ -250,62 +192,42 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
      continue;
    }

-    int32_t rows = pDataBlock->info.rows;
+    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE,};

-    SSubmitTbData tbData = {0};
+    int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
+    tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray);

-    if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow *)))) {
-      code = terrno;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-    tbData.suid = suid;
-    tbData.uid = 0;  // uid is assigned by vnode
-    tbData.sver = pTSchema->version;
+    {
+      uint64_t groupId = pDataBlock->info.id.groupId;

-    if (createTb) {
-      tbData.pCreateTbReq = taosArrayGetP(createTbArray, i);
-      if (tbData.pCreateTbReq) tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
-    }
-
-    if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) {
-      taosArrayDestroy(tbData.aRowP);
-      code = terrno;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    for (int32_t j = 0; j < rows; ++j) {
-      taosArrayClear(pVals);
-      for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
-        const STColumn  *pCol = &pTSchema->columns[k];
-        SColumnInfoData *pColData = taosArrayGet(pDataBlock->pDataBlock, k);
-        if (colDataIsNull_s(pColData, j)) {
-          SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
-          taosArrayPush(pVals, &cv);
-        } else {
-          void *data = colDataGetData(pColData, j);
-          if (IS_STR_DATA_TYPE(pCol->type)) {
-            SValue  sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)};  // address copy, no value
-            SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
-            taosArrayPush(pVals, &cv);
-          } else {
-            SValue sv;
-            memcpy(&sv.val, data, tDataTypes[pCol->type].bytes);
-            SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
-            taosArrayPush(pVals, &cv);
-          }
-        }
-      }
-      SRow *pRow = NULL;
-      if ((code = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) {
-        tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
-        TSDB_CHECK_CODE(code, lino, _exit);
-      }
-      taosArrayPush(tbData.aRowP, &pRow);
+      int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
+      if (index == NULL) {  // no data yet, append it
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
+        if (code != TSDB_CODE_SUCCESS) {
+          continue;
        }

        taosArrayPush(pReq->aSubmitTbData, &tbData);
+
+        int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1;
+        taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
+      } else {
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
+        if (code != TSDB_CODE_SUCCESS) {
+          continue;
        }

+        SSubmitTbData *pExisted = taosArrayGet(pReq->aSubmitTbData, *index);
+        code = doMergeExistedRows(pExisted, &tbData, "id");
+        if (code != TSDB_CODE_SUCCESS) {
+          continue;
+        }
+      }
+    }
+  }
+
+  taosHashCleanup(pTableIndexMap);
+
  // encode
  tEncodeSize(tEncodeSubmitReq, pReq, len, code);
  if (TSDB_CODE_SUCCESS == code) {
@ -327,11 +249,10 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
    }
    tEncoderClear(&encoder);
  }
+
 _exit:
-  taosArrayDestroy(createTbArray);
  taosArrayDestroy(tagArray);
-  taosArrayDestroy(pVals);
-  if (pReq) {
+  if (pReq != NULL) {
    tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
    taosMemoryFree(pReq);
  }
@ -442,8 +363,8 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char
  void           *pSubmitReq = NULL;
  int32_t         contLen = 0;

-  code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true,
-                          pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen);
+  code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, pTsmaStat->pTSma->dstTbUid,
+                          pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen);
  TSDB_CHECK_CODE(code, lino, _exit);

  if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) {
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@ -20,6 +20,12 @@ typedef struct {
  int8_t inited;
 } STqMgmt;

+typedef struct STaskUpdateEntry {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t transId;
+} STaskUpdateEntry;
+
 static STqMgmt tqMgmt = {0};

 // 0: not init
@ -820,28 +826,29 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
  }

  // sink
-  if (pTask->outputInfo.type == TASK_OUTPUT__SMA) {
-    pTask->outputInfo.smaSink.vnode = pTq->pVnode;
-    pTask->outputInfo.smaSink.smaSink = smaHandleRes;
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
-    pTask->outputInfo.tbSink.vnode = pTq->pVnode;
-    pTask->outputInfo.tbSink.tbSinkFunc = tqSinkDataIntoDstTable;
+  STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
+  if (pOutputInfo->type == TASK_OUTPUT__SMA) {
+    pOutputInfo->smaSink.vnode = pTq->pVnode;
+    pOutputInfo->smaSink.smaSink = smaHandleRes;
+  } else if (pOutputInfo->type == TASK_OUTPUT__TABLE) {
+    pOutputInfo->tbSink.vnode = pTq->pVnode;
+    pOutputInfo->tbSink.tbSinkFunc = tqSinkDataIntoDstTable;

    int32_t   ver1 = 1;
    SMetaInfo info = {0};
-    code = metaGetInfo(pTq->pVnode->pMeta, pTask->outputInfo.tbSink.stbUid, &info, NULL);
+    code = metaGetInfo(pTq->pVnode->pMeta, pOutputInfo->tbSink.stbUid, &info, NULL);
    if (code == TSDB_CODE_SUCCESS) {
      ver1 = info.skmVer;
    }

-    SSchemaWrapper* pschemaWrapper = pTask->outputInfo.tbSink.pSchemaWrapper;
-    pTask->outputInfo.tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
-    if (pTask->outputInfo.tbSink.pTSchema == NULL) {
+    SSchemaWrapper* pschemaWrapper = pOutputInfo->tbSink.pSchemaWrapper;
+    pOutputInfo->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
+    if (pOutputInfo->tbSink.pTSchema == NULL) {
      return -1;
    }

-    pTask->outputInfo.tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
-    tSimpleHashSetFreeFp(pTask->outputInfo.tbSink.pTblInfo, freePtr);
+    pOutputInfo->tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
+    tSimpleHashSetFreeFp(pOutputInfo->tbSink.pTblInfo, freePtr);
  }

  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -849,11 +856,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
    pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId);
  }

-  // reset the task status from unfinished transaction
-  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-    tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
-    pTask->status.taskStatus = TASK_STATUS__NORMAL;
-  }
+//  // reset the task status from unfinished transaction
+//  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+//    tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
+//    pTask->status.taskStatus = TASK_STATUS__READY;
+//  }

  streamTaskResetUpstreamStageInfo(pTask);
  streamSetupScheduleTrigger(pTask);
@ -866,20 +873,23 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
           pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
  }

+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
  if (pTask->info.fillHistory) {
    tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
           " nextProcessVer:%" PRId64
           " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
           vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-           pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
-           pTask->info.fillHistory, (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
+           pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
+           (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
  } else {
    tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
           " nextProcessVer:%" PRId64
           " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
           vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-           pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
-           pTask->info.fillHistory, (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
+           pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
+           (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
  }

  return 0;
@ -921,9 +931,10 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
      rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
      streamMetaReleaseTask(pMeta, pTask);

-      const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+      char* p = NULL;
+      streamTaskGetStatus(pTask, &p);
      tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d",
-              pTask->id.idStr, pStatus, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+              pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
    } else {
      rsp.status = TASK_DOWNSTREAM_NOT_READY;
      tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64
@ -957,6 +968,12 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) {
  tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d",
          rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);

+  if (!vnodeIsRoleLeader(pTq->pVnode)) {
+    tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId,
+            rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId);
+    return code;
+  }
+
  SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.streamId, rsp.upstreamTaskId);
  if (pTask == NULL) {
    tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
@ -1026,11 +1043,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
      SStreamTask* p = streamMetaAcquireTask(pStreamMeta, streamId, taskId);

      bool restored = pTq->pVnode->restored;
-      if (p != NULL && restored) {
-        p->execInfo.init = taosGetTimestampMs();
-        tqDebug("s-task:%s set the init ts:%"PRId64, p->id.idStr, p->execInfo.init);
-
-        streamTaskCheckDownstream(p);
+      if (p != NULL && restored && p->info.fillHistory == 0) {
+        EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+        streamTaskHandleEvent(p->status.pSM, event);
      } else if (!restored) {
        tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId);
      }
@ -1064,7 +1079,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {

  // do recovery step1
  const char* id = pTask->id.idStr;
-  const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+  char*       pStatus = NULL;
+  streamTaskGetStatus(pTask, &pStatus);

  // avoid multi-thread exec
  while(1) {
@ -1117,8 +1133,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
  streamScanHistoryData(pTask);

  double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
-  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-    int8_t status = streamTaskSetSchedStatusInActive(pTask);
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE) {
+    int8_t status = streamTaskSetSchedStatusInactive(pTask);
    tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status);

    atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
@ -1127,12 +1143,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
  }

  // the following procedure should be executed, no matter status is stop/pause or not
-  tqDebug("s-task:%s scan-history stage(step 1) ended, elapsed time:%.2fs", id, el);
+  tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, el);

  if (pTask->info.fillHistory) {
    SVersionRange* pRange = NULL;
    SStreamTask*   pStreamTask = NULL;
-    bool           done = false;

    // 1. get the related stream task
    pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
@ -1151,10 +1166,10 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
    }

    ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE);
-
+#if 0
    // 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the
    // stream task get ready for scan history data
-    while (pStreamTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+    while (streamTaskGetStatus(pStreamTask, NULL) == TASK_STATUS__SCAN_HISTORY) {
      tqDebug(
          "s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms",
          id, pTask->info.taskLevel, pStreamTask->id.idStr, streamGetTaskStatusStr(pStreamTask->status.taskStatus));
@ -1199,6 +1214,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
      pStreamTask->status.keepTaskStatus = status;
      pStreamTask->status.taskStatus = TASK_STATUS__HALT;

+      // wal scan not start yet, reset it to be the start position
      nextProcessedVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader);
      if (nextProcessedVer == -1) {
        nextProcessedVer = pStreamTask->dataRange.range.maxVer + 1;
@ -1211,21 +1227,26 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
      taosThreadMutexUnlock(&pStreamTask->lock);
      break;
    }
+#endif
+
+    streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
+    int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer;

    // if it's an source task, extract the last version in wal.
    pRange = &pTask->dataRange.range;
-    done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
+    bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
    pTask->execInfo.step2Start = taosGetTimestampMs();

    if (done) {
      qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, 0.0);
      streamTaskPutTranstateIntoInputQ(pTask);
+//      streamTaskRestoreStatus(pTask);

-      if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-        pTask->status.keepTaskStatus = TASK_STATUS__NORMAL;
-        qDebug("s-task:%s prev status is %s, update the kept status to be:%s when after step 2", id,
-               streamGetTaskStatusStr(TASK_STATUS__PAUSE), streamGetTaskStatusStr(pTask->status.keepTaskStatus));
-      }
+//      if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+//        pTask->status.keepTaskStatus = TASK_STATUS__READY;
+//        qDebug("s-task:%s prev status is %s, update the kept status to be:%s when after step 2", id,
+//               streamGetTaskStatusStr(TASK_STATUS__PAUSE), streamGetTaskStatusStr(pTask->status.keepTaskStatus));
+//      }

      streamExecTask(pTask);  // exec directly
    } else {
@ -1245,35 +1266,24 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
      tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer,
              pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE);

-      /*int8_t status = */streamTaskSetSchedStatusInActive(pTask);
-
-      // the fill-history task starts to process data in wal, let's set it status to be normal now
-      if (pTask->info.fillHistory == 1 && !streamTaskShouldStop(&pTask->status)) {
-        streamSetStatusNormal(pTask);
-      }
+      /*int8_t status = */streamTaskSetSchedStatusInactive(pTask);

+      // now the fill-history task starts to scan data from wal files.
+      streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
      tqScanWalAsync(pTq, false);
    }
    streamMetaReleaseTask(pMeta, pStreamTask);

  } else {
    STimeWindow* pWindow = &pTask->dataRange.window;
+    ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));

-    if (pTask->hTaskInfo.id.taskId == 0) {
-      *pWindow = (STimeWindow){INT64_MIN, INT64_MAX};
+    // Not update the fill-history time window until the state transfer is completed if the related fill-history task
+    // exists.
    tqDebug(
-          "s-task:%s scan-history in stream time window completed, no related fill-history task, reset the time "
-          "window:%" PRId64 " - %" PRId64,
-          id, pWindow->skey, pWindow->ekey);
-      qStreamInfoResetTimewindowFilter(pTask->exec.pExecutor);
-    } else {
-      // when related fill-history task exists, update the fill-history time window only when the
-      // state transfer is completed.
-      tqDebug(
-          "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, start "
-          "ver:%" PRId64 ", window:%" PRId64 " - %" PRId64,
+        "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, startVer:%" PRId64
+        ", window:%" PRId64 " - %" PRId64,
        id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey);
-    }

    code = streamTaskScanHistoryDataComplete(pTask);
  }
@ -1352,7 +1362,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
  int32_t vgId = TD_VID(pTq->pVnode);

  if (taskId == STREAM_EXEC_TASK_STATUS_CHECK_ID) {
-    tqCheckAndRunStreamTask(pTq);
+    tqStartStreamTask(pTq);
    return 0;
  }

@ -1362,17 +1372,16 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
  }

  SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId);
-  if (pTask != NULL) {
-    // even in halt status, the data in inputQ must be processed
-    int8_t st = pTask->status.taskStatus;
-    if (st == TASK_STATUS__NORMAL || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK) {
+  if (pTask != NULL) { // even in halt status, the data in inputQ must be processed
+    char* p = NULL;
+    if (streamTaskReadyToRun(pTask, &p)) {
      tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr,
              pTask->chkInfo.nextProcessVer);
      streamExecTask(pTask);
    } else {
-      int8_t status = streamTaskSetSchedStatusInActive(pTask);
+      int8_t status = streamTaskSetSchedStatusInactive(pTask);
      tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
-              pTask->id.idStr, streamGetTaskStatusStr(st), status);
+              pTask->id.idStr, p, status);
    }

    streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@ -1446,7 +1455,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
  if (pTask != NULL) {
    // drop the related fill-history task firstly
-    if (pTask->hTaskInfo.id.taskId != 0) {
+    if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
      STaskId* pHTaskId = &pTask->hTaskInfo.id;
      streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId);
      tqDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId);
@ -1486,7 +1495,7 @@ int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
  streamTaskPause(pTask, pMeta);

  SStreamTask* pHistoryTask = NULL;
-  if (pTask->hTaskInfo.id.taskId != 0) {
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
    pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId);
    if (pHistoryTask == NULL) {
      tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
@ -1514,17 +1523,19 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
    return -1;
  }

-  // todo: handle the case: resume from halt to pause/ from halt to normal/ from pause to normal
-  streamTaskResume(pTask, pTq->pStreamMeta);
+  streamTaskResume(pTask);
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);

  int32_t level = pTask->info.taskLevel;
  if (level == TASK_LEVEL__SINK) {
+    if (status == TASK_STATUS__UNINIT) {
+
+    }
    streamMetaReleaseTask(pTq->pStreamMeta, pTask);
    return 0;
  }

-  int8_t  status = pTask->status.taskStatus;
-  if (status == TASK_STATUS__NORMAL || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
+  if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
    // no lock needs to secure the access of the version
    if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) {
      // discard all the data  when the stream task is suspended.
@ -1537,14 +1548,18 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
              vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus);
    }

-    if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory &&
-        pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+    if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
      streamStartScanHistoryAsync(pTask, igUntreated);
-    } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputInfo.queue) == 0)) {
+    } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
      tqScanWalAsync(pTq, false);
    } else {
      streamSchedExec(pTask);
    }
+  } else if (status == TASK_STATUS__UNINIT) {
+    if (pTask->info.fillHistory == 0) {
+      EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+      streamTaskHandleEvent(pTask->status.pSM, event);
+    }
  }

  streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@ -1683,7 +1698,6 @@ FAIL:
  return -1;
 }

-// todo error code cannot be return, since this is invoked by an mnode-launched transaction.
 int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) {
  int32_t      vgId = TD_VID(pTq->pVnode);
  SStreamMeta* pMeta = pTq->pStreamMeta;
@ -1694,7 +1708,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
  // disable auto rsp to mnode
  pRsp->info.handle = NULL;

-  // todo: add counter to make sure other tasks would not be trapped in checkpoint state
  SStreamCheckpointSourceReq req = {0};
  if (!vnodeIsRoleLeader(pTq->pVnode)) {
    tqDebug("vgId:%d not leader, ignore checkpoint-source msg", vgId);
@ -1725,6 +1738,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
  }
  tDecoderClear(&decoder);

+  // todo handle failure to reset from checkpoint procedure
  SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
  if (pTask == NULL) {
    tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. it may have been destroyed already", vgId,
@ -1735,6 +1749,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
    return TSDB_CODE_SUCCESS;
  }

+  // todo handle failure to reset from checkpoint procedure
  // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req.
  if (pTask->status.downstreamReady != 1) {
    pTask->chkInfo.failedId = req.checkpointId;   // record the latest failed checkpoint id
@ -1750,8 +1765,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
    return TSDB_CODE_SUCCESS;
  }

+  // todo save the checkpoint failed info
  taosThreadMutexLock(&pTask->lock);
-  if (pTask->status.taskStatus == TASK_STATUS__HALT) {
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
+
+  if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {
    qError("s-task:%s not ready for checkpoint, since it is halt, ignore this checkpoint:%" PRId64 ", set it failure",
           pTask->id.idStr, req.checkpointId);
    taosThreadMutexUnlock(&pTask->lock);
@ -1832,7 +1850,6 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
  char*        msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
  int32_t      len = pMsg->contLen - sizeof(SMsgHead);
  SRpcMsg      rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
-  bool         allStopped = false;

  SStreamTaskNodeUpdateMsg req = {0};

@ -1863,17 +1880,43 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
  }

  SStreamTask* pTask = *ppTask;
-  tqDebug("s-task:%s receive nodeEp update msg from mnode", pTask->id.idStr);

+  if (pMeta->updateInfo.transId != req.transId) {
+    pMeta->updateInfo.transId = req.transId;
+    tqDebug("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId);
+    // info needs to be kept till the new trans to update the nodeEp arrived.
+    taosHashClear(pMeta->updateInfo.pTasks);
+  } else {
+    tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId);
+  }
+
+  STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId};
+  void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
+  if (exist != NULL) {
+    tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId,
+            req.transId);
+    rsp.code = TSDB_CODE_SUCCESS;
+    taosWUnLockLatch(&pMeta->lock);
+    taosArrayDestroy(req.pNodeList);
+    return rsp.code;
+  }
+
+  taosWUnLockLatch(&pMeta->lock);
+
+  // the following two functions should not be executed within the scope of meta lock to avoid deadlock
  streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
-  streamSetStatusNormal(pTask);
+  streamTaskResetStatus(pTask);
+
+  // continue after lock the meta again
+  taosWLockLatch(&pMeta->lock);

  SStreamTask** ppHTask = NULL;
-  if (pTask->hTaskInfo.id.taskId != 0) {
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
    ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id));
    if (ppHTask == NULL || *ppHTask == NULL) {
      tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already",
              pMeta->vgId, req.taskId);
+      CLEAR_RELATED_FILLHISTORY_TASK(pTask);
    } else {
      tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr);
      streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList);
@ -1892,12 +1935,14 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
  }

  streamTaskStop(pTask);
-  taosHashPut(pMeta->pUpdateTaskSet, &pTask->id, sizeof(pTask->id), NULL, 0);
+
+  // keep the already handled info
+  taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0);

  if (ppHTask != NULL) {
    streamTaskStop(*ppHTask);
    tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr);
-    taosHashPut(pMeta->pUpdateTaskSet, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0);
+    taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0);
  } else {
    tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr);
  }
@ -1906,25 +1951,30 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {

  // possibly only handle the stream task.
  int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
-  int32_t updateTasks = taosHashGetSize(pMeta->pUpdateTaskSet);
+  int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks);

-  pMeta->startInfo.startedAfterNodeUpdate = 1;
+  pMeta->startInfo.startAllTasksFlag = 1;

  if (updateTasks < numOfTasks) {
    tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId,
            updateTasks, (numOfTasks - updateTasks));
    taosWUnLockLatch(&pMeta->lock);
  } else {
-    taosHashClear(pMeta->pUpdateTaskSet);
-
    if (!pTq->pVnode->restored) {
      tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId);
-      pMeta->startInfo.startedAfterNodeUpdate = 0;
+      pMeta->startInfo.startAllTasksFlag = 0;
      taosWUnLockLatch(&pMeta->lock);
    } else {
      tqDebug("vgId:%d tasks are all updated and stopped, restart them", vgId);
-
      terrno = 0;
+      taosWUnLockLatch(&pMeta->lock);
+
+      while (streamMetaTaskInTimer(pMeta)) {
+        qDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
+        taosMsleep(100);
+      }
+
+      taosWLockLatch(&pMeta->lock);
      int32_t code = streamMetaReopen(pMeta);
      if (code != 0) {
        tqError("vgId:%d failed to reopen stream meta", vgId);
@ -1942,8 +1992,8 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {

      if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) {
        vInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
-        tqStartStreamTasks(pTq);
-        tqCheckAndRunStreamTaskAsync(pTq);
+        tqResetStreamTaskStatus(pTq);
+        tqLaunchStreamTaskAsync(pTq);
      } else {
        vInfo("vgId:%d, follower node not start stream tasks", vgId);
      }
@ -1970,10 +2020,10 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
  tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);

  // clear flag set during do checkpoint, and open inputQ for all upstream tasks
-  if (pTask->status.taskStatus == TASK_STATUS__CK) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) {
    streamTaskClearCheckInfo(pTask);
    taosArrayClear(pTask->pReadyMsgList);
-    streamSetStatusNormal(pTask);
+    streamTaskSetStatusReady(pTask);
  }

  streamMetaReleaseTask(pMeta, pTask);
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@ -39,7 +39,7 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) {
  int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta);
  taosRUnLockLatch(&pTq->pStreamMeta->lock);

-  tqDebug("handle submit, restore:%d, numOfTasks:%d", pTq->pVnode->restored, numOfTasks);
+//  tqTrace("vgId:%d handle submit, restore:%d, numOfTasks:%d", TD_VID(pTq->pVnode), pTq->pVnode->restored, numOfTasks);

  // push data for stream processing:
  // 1. the vnode has already been restored.
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@ -13,6 +13,7 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include <common/tmsg.h>
 #include "tcommon.h"
 #include "tmsg.h"
 #include "tq.h"
@ -28,19 +29,19 @@ static bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks);
 static int32_t tsAscendingSortFn(const void* p1, const void* p2);
 static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName,
                                  SSubmitTbData* pTableData);
-static int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock,
-                                      SSubmitTbData* pTableData);
 static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
                                       int64_t suid);
 static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks);
 static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen);
-static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id);
+static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id);
 static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo,
                                        const char* dstTableName, int64_t* uid);
 static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id);
-static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
 static bool    isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
-static SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock);
+static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags);
+static SArray* createDefaultTagColName();
+static void    setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
+                                   int64_t gid);

 int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
                         const char* pIdStr) {
@ -138,61 +139,68 @@ static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
  return TSDB_CODE_SUCCESS;
 }

+int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
+  pCreateTableReq->flags = 0;
+  pCreateTableReq->type = TSDB_CHILD_TABLE;
+  pCreateTableReq->ctb.suid = suid;
+
+  // set super table name
+  SName name = {0};
+  tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+  pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
+
+  pCreateTableReq->ctb.tagNum = numOfTags;
+  return TSDB_CODE_SUCCESS;
+}
+
+SArray* createDefaultTagColName() {
+  SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
+  char    tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
+  taosArrayPush(pTagColNameList, tagNameStr);
+  return pTagColNameList;
+}
+
+void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
+                                int64_t gid) {
+  if (pDataBlock->info.parTbName[0]) {
+    pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
+  } else {
+    pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid);
+  }
+}
+
 static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
                                            int64_t suid) {
  tqDebug("s-task:%s build create table msg", pTask->id.idStr);

  STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
  int32_t   rows = pDataBlock->info.rows;
-  SArray*   tagArray = NULL;
+  SArray*   tagArray = taosArrayInit(4, sizeof(STagVal));;
  int32_t   code = 0;

  SVCreateTbBatchReq reqs = {0};
-
  SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
  if (NULL == reqs.pArray) {
+    tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno));
    goto _end;
  }

  for (int32_t rowId = 0; rowId < rows; rowId++) {
    SVCreateTbReq* pCreateTbReq = &((SVCreateTbReq){0});

-    // set const
-    pCreateTbReq->flags = 0;
-    pCreateTbReq->type = TSDB_CHILD_TABLE;
-    pCreateTbReq->ctb.suid = suid;
-
-    // set super table name
-    SName name = {0};
-    tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-    pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));  // taosStrdup(stbFullName);
-
-    // set tag content
    int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
-    if (size == 2) {
-      tagArray = taosArrayInit(1, sizeof(STagVal));
-      if (!tagArray) {
-        tdDestroySVCreateTbReq(pCreateTbReq);
-        goto _end;
-      }
+    int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);

+    initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
+    taosArrayClear(tagArray);
+
+    if (size == 2) {
      STagVal tagVal = {
          .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};

      taosArrayPush(tagArray, &tagVal);
-
-      // set tag name
-      SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-      char    tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
-      taosArrayPush(tagName, tagNameStr);
-      pCreateTbReq->ctb.tagName = tagName;
+      pCreateTbReq->ctb.tagName = createDefaultTagColName();
    } else {
-      tagArray = taosArrayInit(size - 1, sizeof(STagVal));
-      if (!tagArray) {
-        tdDestroySVCreateTbReq(pCreateTbReq);
-        goto _end;
-      }
-
      for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
        SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);

@ -209,29 +217,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
        taosArrayPush(tagArray, &tagVal);
      }
    }
-    pCreateTbReq->ctb.tagNum = TMAX(size - UD_TAG_COLUMN_INDEX, 1);

-    STag* pTag = NULL;
-    tTagNew(tagArray, 1, false, &pTag);
+    tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
    tagArray = taosArrayDestroy(tagArray);
-    if (pTag == NULL) {
+    if (pCreateTbReq->ctb.pTag == NULL) {
      tdDestroySVCreateTbReq(pCreateTbReq);
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _end;
    }

-    pCreateTbReq->ctb.pTag = (uint8_t*)pTag;
-
-    // set table name
-    if (!pDataBlock->info.parTbName[0]) {
+    uint64_t gid = pDataBlock->info.id.groupId;
+    if (taosArrayGetSize(pDataBlock->pDataBlock) > UD_GROUPID_COLUMN_INDEX) {
      SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX);

+      // todo remove this
      void* pGpIdData = colDataGetData(pGpIdColInfo, rowId);
-      pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData);
-    } else {
-      pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
+      ASSERT(gid == *(int64_t*)pGpIdData);
    }

+    setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid);
+
    taosArrayPush(reqs.pArray, pCreateTbReq);
    tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
  }
@ -330,6 +335,9 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c

  tqTrace("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id,
          (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (pNew->pCreateTbReq != NULL));
+
+  tdDestroySVCreateTbReq(pNew->pCreateTbReq);
+  taosMemoryFree(pNew->pCreateTbReq);
  return TSDB_CODE_SUCCESS;
 }

@ -390,60 +398,33 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
  return true;
 }

-SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock) {
-  char* ctbName = pDataBlock->info.parTbName;
-
-  SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq));
+SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
+                                       SSDataBlock* pDataBlock, SArray* pTagArray) {
+  SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
  if (pCreateTbReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

-  // set tag content
-  SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
-  if (tagArray == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tdDestroySVCreateTbReq(pCreateTbReq);
-    taosMemoryFreeClear(pCreateTbReq);
-    return NULL;
-  }
-
-  // set const
-  pCreateTbReq->flags = 0;
-  pCreateTbReq->type = TSDB_CHILD_TABLE;
-  pCreateTbReq->ctb.suid = suid;
-
-  // set super table name
-  SName name = {0};
-  tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
+  taosArrayClear(pTagArray);
+  initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);

  STagVal tagVal = { .cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
-  taosArrayPush(tagArray, &tagVal);
-  pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
+  taosArrayPush(pTagArray, &tagVal);

-  STag* pTag = NULL;
-  tTagNew(tagArray, 1, false, &pTag);
-  taosArrayDestroy(tagArray);
+  tTagNew(pTagArray, 1, false, (STag**) &pCreateTbReq->ctb.pTag);

-  if (pTag == NULL) {
+  if (pCreateTbReq->ctb.pTag == NULL) {
    tdDestroySVCreateTbReq(pCreateTbReq);
    taosMemoryFreeClear(pCreateTbReq);
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

-  pCreateTbReq->ctb.pTag = (uint8_t*)pTag;
-
-  // set tag name
-  SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-  char k[TSDB_COL_NAME_LEN] = "group_id";
-  taosArrayPush(tagName, k);
-
-  pCreateTbReq->ctb.tagName = tagName;
+  pCreateTbReq->ctb.tagName = createDefaultTagColName();

  // set table name
-  pCreateTbReq->name = taosStrdup(ctbName);
+  setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId);
  return pCreateTbReq;
 }

@ -514,7 +495,7 @@ int32_t tsAscendingSortFn(const void* p1, const void* p2) {
  }
 }

-int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
+int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
  int32_t numOfRows = pDataBlock->info.rows;
  int32_t code = TSDB_CODE_SUCCESS;

@ -592,7 +573,7 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
  const char* id = pTask->id.idStr;

  while (pTableSinkInfo->uid == 0) {
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask)) {
      tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName);
      return TSDB_CODE_STREAM_EXEC_CANCELLED;
    }
@ -693,8 +674,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat

      tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName);

+      SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
+
      pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
-      pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock);
+      pTableData->pCreateTbReq =
+          buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray);
+      taosArrayDestroy(pTagArray);
+
      if (pTableData->pCreateTbReq == NULL) {
        tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno));
        taosMemoryFree(pTableSinkInfo);
@ -724,17 +710,16 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
  return TSDB_CODE_SUCCESS;
 }

-int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock,
-                               SSubmitTbData* pTableData) {
+int32_t setDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
+                               SSubmitTbData* pTableData, const char* id) {
  int32_t numOfRows = pDataBlock->info.rows;
-  const char* id = pTask->id.idStr;

  tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64,
-          id, blockIndex + 1, numOfRows, pTask->outputInfo.tbSink.stbUid);
+          id, blockIndex + 1, numOfRows, suid);
  char* dstTableName = pDataBlock->info.parTbName;

  // convert all rows
-  int32_t code = doConvertRows(pTableData, pTask->outputInfo.tbSink.pTSchema, pDataBlock, id);
+  int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, id);
  if (code != TSDB_CODE_SUCCESS) {
    tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno));
    return code;
@ -773,7 +758,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
            numOfBlocks);

    for(int32_t i = 0; i < numOfBlocks; ++i) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
        return;
      }

@ -800,7 +785,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
          continue;
        }

-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
        if (code != TSDB_CODE_SUCCESS) {
          continue;
        }
@ -823,7 +808,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {

    bool hasSubmit = false;
    for (int32_t i = 0; i < numOfBlocks; i++) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
        taosHashCleanup(pTableIndexMap);
        tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
        return;
@ -847,7 +832,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
          continue;
        }

-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
        if (code != TSDB_CODE_SUCCESS) {
          continue;
        }
@ -857,7 +842,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
        int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
        taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
      } else {
-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
        if (code != TSDB_CODE_SUCCESS) {
          continue;
        }
--- a/source/dnode/vnode/src/tq/tqStreamTask.c
+++ b/source/dnode/vnode/src/tq/tqStreamTask.c
@ -60,7 +60,7 @@ int32_t tqScanWal(STQ* pTq) {
  return 0;
 }

-int32_t tqCheckAndRunStreamTask(STQ* pTq) {
+int32_t tqStartStreamTask(STQ* pTq) {
  int32_t      vgId = TD_VID(pTq->pVnode);
  SStreamMeta* pMeta = pTq->pStreamMeta;

@ -92,19 +92,19 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
    }

    if (pTask->status.downstreamReady == 1) {
+      if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
        tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task",
                pTask->id.idStr);
        streamLaunchFillHistoryTask(pTask);
+      }
+
+      streamMetaUpdateTaskReadyInfo(pTask);
      streamMetaReleaseTask(pMeta, pTask);
      continue;
    }

-    pTask->execInfo.init = taosGetTimestampMs();
-    tqDebug("s-task:%s start check downstream tasks, set the init ts:%"PRId64, pTask->id.idStr, pTask->execInfo.init);
-
-    streamSetStatusNormal(pTask);
-    streamTaskCheckDownstream(pTask);
-
+    EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+    streamTaskHandleEvent(pTask->status.pSM, event);
    streamMetaReleaseTask(pMeta, pTask);
  }

@ -112,9 +112,9 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
  return 0;
 }

-int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq) {
-  int32_t      vgId = TD_VID(pTq->pVnode);
+int32_t tqLaunchStreamTaskAsync(STQ* pTq) {
  SStreamMeta* pMeta = pTq->pStreamMeta;
+  int32_t      vgId = pMeta->vgId;

  int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
  if (numOfTasks == 0) {
@ -228,7 +228,7 @@ int32_t tqStopStreamTasks(STQ* pTq) {
  return 0;
 }

-int32_t tqStartStreamTasks(STQ* pTq) {
+int32_t tqResetStreamTaskStatus(STQ* pTq) {
  SStreamMeta* pMeta = pTq->pStreamMeta;
  int32_t      vgId = TD_VID(pTq->pVnode);
  int32_t      numOfTasks = taosArrayGetSize(pMeta->pTaskList);
@ -243,11 +243,7 @@ int32_t tqStartStreamTasks(STQ* pTq) {

    STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId};
    SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
-
-    int8_t status = (*pTask)->status.taskStatus;
-    if (status == TASK_STATUS__STOP && (*pTask)->info.fillHistory != 1) {
-      streamSetStatusNormal(*pTask);
-    }
+    streamTaskResetStatus(*pTask);
  }

  return 0;
@ -328,15 +324,17 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
  }

  // not in ready state, do not handle the data from wal
-  int32_t status = pTask->status.taskStatus;
-  if (status != TASK_STATUS__NORMAL) {
-    tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
+//  int32_t status = pTask->status.taskStatus;
+  char* p = NULL;
+  int32_t status = streamTaskGetStatus(pTask, &p);
+  if (streamTaskGetStatus(pTask, &p) != TASK_STATUS__READY) {
+    tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, p);
    return false;
  }

  // fill-history task has entered into the last phase, no need to anything
  if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
-    ASSERT(status == TASK_STATUS__NORMAL);
+    ASSERT(status == TASK_STATUS__READY);
    // the maximum version of data in the WAL has reached already, the step2 is done
    tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
            pTask->dataRange.range.maxVer);
@ -344,13 +342,13 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
  }

  // check if input queue is full or not
-  if (streamQueueIsFull(pTask->inputInfo.queue)) {
+  if (streamQueueIsFull(pTask->inputq.queue)) {
    tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr);
    return false;
  }

  // the input queue of downstream task is full, so the output is blocked, stopped for a while
-  if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+  if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
    tqDebug("s-task:%s inputQ is blocked, do nothing", pTask->id.idStr);
    return false;
  }
@ -444,14 +442,15 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
      continue;
    }

-    int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputInfo.queue);
+    int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
    int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX;

    taosThreadMutexLock(&pTask->lock);

-    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
-    if (pTask->status.taskStatus != TASK_STATUS__NORMAL) {
-      tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pStatus);
+    char* p = NULL;
+    ETaskStatus status = streamTaskGetStatus(pTask, &p);
+    if (status != TASK_STATUS__READY) {
+      tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, p);
      taosThreadMutexUnlock(&pTask->lock);
      streamMetaReleaseTask(pStreamMeta, pTask);
      continue;
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@ -131,7 +131,7 @@ int32_t tsdbBegin(STsdb *pTsdb) {
  TSDB_CHECK_CODE(code, lino, _exit);

  // lock
-  if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) {
+  if ((code = taosThreadMutexLock(&pTsdb->mutex))) {
    code = TAOS_SYSTEM_ERROR(code);
    TSDB_CHECK_CODE(code, lino, _exit);
  }
@ -139,7 +139,7 @@ int32_t tsdbBegin(STsdb *pTsdb) {
  pTsdb->mem = pMemTable;

  // unlock
-  if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) {
+  if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) {
    code = TAOS_SYSTEM_ERROR(code);
    TSDB_CHECK_CODE(code, lino, _exit);
  }
@ -152,11 +152,11 @@ _exit:
 }

 int32_t tsdbPrepareCommit(STsdb *pTsdb) {
-  taosThreadRwlockWrlock(&pTsdb->rwLock);
+  taosThreadMutexLock(&pTsdb->mutex);
  ASSERT(pTsdb->imem == NULL);
  pTsdb->imem = pTsdb->mem;
  pTsdb->mem = NULL;
-  taosThreadRwlockUnlock(&pTsdb->rwLock);
+  taosThreadMutexUnlock(&pTsdb->mutex);

  return 0;
 }
@ -171,9 +171,9 @@ int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) {

  // check
  if (pMemTable->nRow == 0 && pMemTable->nDel == 0) {
-    taosThreadRwlockWrlock(&pTsdb->rwLock);
+    taosThreadMutexLock(&pTsdb->mutex);
    pTsdb->imem = NULL;
-    taosThreadRwlockUnlock(&pTsdb->rwLock);
+    taosThreadMutexUnlock(&pTsdb->mutex);

    tsdbUnrefMemTable(pMemTable, NULL, true);
    goto _exit;
@ -501,6 +501,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
  int32_t    lino = 0;
  STsdb     *pTsdb = pCommitter->pTsdb;
  SDFileSet *pRSet = NULL;
+
  // memory
  pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision);
  pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec());
@ -798,6 +799,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
  int32_t    lino = 0;
  STsdb     *pTsdb = pCommitter->pTsdb;
  SMemTable *pMemTable = pTsdb->imem;
+
  // commit file data start
  code = tsdbCommitFileDataStart(pCommitter);
  TSDB_CHECK_CODE(code, lino, _exit);
@ -1650,18 +1652,18 @@ int32_t tsdbFinishCommit(STsdb *pTsdb) {
  SMemTable *pMemTable = pTsdb->imem;

  // lock
-  taosThreadRwlockWrlock(&pTsdb->rwLock);
+  taosThreadMutexLock(&pTsdb->mutex);

  code = tsdbFSCommit(pTsdb);
  if (code) {
-    taosThreadRwlockUnlock(&pTsdb->rwLock);
+    taosThreadMutexUnlock(&pTsdb->mutex);
    TSDB_CHECK_CODE(code, lino, _exit);
  }

  pTsdb->imem = NULL;

  // unlock
-  taosThreadRwlockUnlock(&pTsdb->rwLock);
+  taosThreadMutexUnlock(&pTsdb->mutex);
  if (pMemTable) {
    tsdbUnrefMemTable(pMemTable, NULL, true);
  }
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@ -368,7 +368,12 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
  int32_t lino = 0;
  STsdb  *tsdb = committer->tsdb;

-  committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
+  int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
+
+  // check if can commit
+  tsdbFSCheckCommit(tsdb, fid);
+
+  committer->ctx->fid = fid;
  committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now);
  tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
                  &committer->ctx->maxKey);
@ -576,11 +581,11 @@ _exit:
 }

 int32_t tsdbPreCommit(STsdb *tsdb) {
-  taosThreadRwlockWrlock(&tsdb->rwLock);
+  taosThreadMutexLock(&tsdb->mutex);
  ASSERT(tsdb->imem == NULL);
  tsdb->imem = tsdb->mem;
  tsdb->mem = NULL;
-  taosThreadRwlockUnlock(&tsdb->rwLock);
+  taosThreadMutexUnlock(&tsdb->mutex);
  return 0;
 }

@ -595,15 +600,13 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
  int64_t    nDel = imem->nDel;

  if (nRow == 0 && nDel == 0) {
-    taosThreadRwlockWrlock(&tsdb->rwLock);
+    taosThreadMutexLock(&tsdb->mutex);
    tsdb->imem = NULL;
-    taosThreadRwlockUnlock(&tsdb->rwLock);
+    taosThreadMutexUnlock(&tsdb->mutex);
    tsdbUnrefMemTable(imem, NULL, true);
  } else {
    SCommitter2 committer[1];

-    tsdbFSCheckCommit(tsdb->pFS);
-
    code = tsdbOpenCommitter(tsdb, info, committer);
    TSDB_CHECK_CODE(code, lino, _exit);

@ -632,14 +635,14 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
  if (tsdb->imem == NULL) goto _exit;

  SMemTable *pMemTable = tsdb->imem;
-  taosThreadRwlockWrlock(&tsdb->rwLock);
+  taosThreadMutexLock(&tsdb->mutex);
  code = tsdbFSEditCommit(tsdb->pFS);
  if (code) {
-    taosThreadRwlockUnlock(&tsdb->rwLock);
+    taosThreadMutexUnlock(&tsdb->mutex);
    TSDB_CHECK_CODE(code, lino, _exit);
  }
  tsdb->imem = NULL;
-  taosThreadRwlockUnlock(&tsdb->rwLock);
+  taosThreadMutexUnlock(&tsdb->mutex);
  tsdbUnrefMemTable(pMemTable, NULL, true);

 _exit:
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c
@ -55,25 +55,11 @@ static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) {
  TARRAY2_INIT(fs[0]->fSetArr);
  TARRAY2_INIT(fs[0]->fSetArrTmp);

-  // background task queue
-  taosThreadMutexInit(fs[0]->mutex, NULL);
-  fs[0]->bgTaskQueue->next = fs[0]->bgTaskQueue;
-  fs[0]->bgTaskQueue->prev = fs[0]->bgTaskQueue;
-
-  taosThreadMutexInit(&fs[0]->commitMutex, NULL);
-  taosThreadCondInit(&fs[0]->canCommit, NULL);
-  fs[0]->blockCommit = false;
-
  return 0;
 }

 static int32_t destroy_fs(STFileSystem **fs) {
  if (fs[0] == NULL) return 0;
-  taosThreadMutexDestroy(&fs[0]->commitMutex);
-  taosThreadCondDestroy(&fs[0]->canCommit);
-  taosThreadMutexDestroy(fs[0]->mutex);
-
-  ASSERT(fs[0]->bgTaskNum == 0);

  TARRAY2_DESTROY(fs[0]->fSetArr, NULL);
  TARRAY2_DESTROY(fs[0]->fSetArrTmp, NULL);
@ -264,10 +250,11 @@ static int32_t apply_commit(STFileSystem *fs) {
    if (fset1 && fset2) {
      if (fset1->fid < fset2->fid) {
        // delete fset1
-        TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove);
+        tsdbTFileSetRemove(fset1);
+        i1++;
      } else if (fset1->fid > fset2->fid) {
        // create new file set with fid of fset2->fid
-        code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1);
+        code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
        if (code) return code;
        code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
        if (code) return code;
@ -282,10 +269,11 @@ static int32_t apply_commit(STFileSystem *fs) {
      }
    } else if (fset1) {
      // delete fset1
-      TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove);
+      tsdbTFileSetRemove(fset1);
+      i1++;
    } else {
      // create new file set with fid of fset2->fid
-      code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1);
+      code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
      if (code) return code;
      code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
      if (code) return code;
@ -512,7 +500,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
        TARRAY2_FOREACH(lvl->fobjArr, fobj) {
          code = tsdbFSDoScanAndFixFile(fs, fobj);
          if (code) {
-            fset->maxVerValid = (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1;
+            fset->maxVerValid =
+                (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1;
            corrupt = true;
          }
        }
@ -592,7 +581,7 @@ static int32_t tsdbFSDupState(STFileSystem *fs) {
  const STFileSet *fset1;
  TARRAY2_FOREACH(src, fset1) {
    STFileSet *fset2;
-    code = tsdbTFileSetInitDup(fs->tsdb, fset1, &fset2);
+    code = tsdbTFileSetInitCopy(fs->tsdb, fset1, &fset2);
    if (code) return code;
    code = TARRAY2_APPEND(dst, fset2);
    if (code) return code;
@ -665,12 +654,6 @@ static int32_t close_file_system(STFileSystem *fs) {
  return 0;
 }

-static int32_t apply_edit(STFileSystem *pFS) {
-  int32_t code = 0;
-  ASSERTS(0, "TODO: Not implemented yet");
-  return code;
-}
-
 static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSet *pSet2) {
  if (pSet1->fid < pSet2->fid) {
    return -1;
@ -710,10 +693,23 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) {
    TSDB_CHECK_CODE(code, lino, _exit);
  }

-  // remove empty file set
+  // remove empty empty stt level and empty file set
  int32_t i = 0;
  while (i < TARRAY2_SIZE(fsetArray)) {
    fset = TARRAY2_GET(fsetArray, i);
+
+    SSttLvl *lvl;
+    int32_t  j = 0;
+    while (j < TARRAY2_SIZE(fset->lvlArr)) {
+      lvl = TARRAY2_GET(fset->lvlArr, j);
+
+      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
+        TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear);
+      } else {
+        j++;
+      }
+    }
+
    if (tsdbTFileSetIsEmpty(fset)) {
      TARRAY2_REMOVE(fsetArray, i, tsdbTFileSetClear);
    } else {
@ -753,13 +749,13 @@ _exit:

 static void tsdbDoWaitBgTask(STFileSystem *fs, STFSBgTask *task) {
  task->numWait++;
-  taosThreadCondWait(task->done, fs->mutex);
+  taosThreadCondWait(task->done, &fs->tsdb->mutex);
  task->numWait--;

  if (task->numWait == 0) {
    taosThreadCondDestroy(task->done);
-    if (task->free) {
-      task->free(task->arg);
+    if (task->destroy) {
+      task->destroy(task->arg);
    }
    taosMemoryFree(task);
  }
@ -770,8 +766,8 @@ static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) {
    taosThreadCondBroadcast(task->done);
  } else {
    taosThreadCondDestroy(task->done);
-    if (task->free) {
-      task->free(task->arg);
+    if (task->destroy) {
+      task->destroy(task->arg);
    }
    taosMemoryFree(task);
  }
@ -780,23 +776,16 @@ static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) {
 int32_t tsdbCloseFS(STFileSystem **fs) {
  if (fs[0] == NULL) return 0;

-  taosThreadMutexLock(fs[0]->mutex);
-  fs[0]->stop = true;
-
-  if (fs[0]->bgTaskRunning) {
-    tsdbDoWaitBgTask(fs[0], fs[0]->bgTaskRunning);
-  }
-  taosThreadMutexUnlock(fs[0]->mutex);
-
+  tsdbFSDisableBgTask(fs[0]);
  close_file_system(fs[0]);
  destroy_fs(fs);
  return 0;
 }

 int64_t tsdbFSAllocEid(STFileSystem *fs) {
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
  int64_t cid = ++fs->neid;
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
  return cid;
 }

@ -837,27 +826,34 @@ _exit:
  return code;
 }

-static int32_t tsdbFSSetBlockCommit(STFileSystem *fs, bool block) {
-  taosThreadMutexLock(&fs->commitMutex);
+static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block) {
  if (block) {
-    fs->blockCommit = true;
+    fset->blockCommit = true;
  } else {
-    fs->blockCommit = false;
-    taosThreadCondSignal(&fs->canCommit);
+    fset->blockCommit = false;
+    if (fset->numWaitCommit > 0) {
+      taosThreadCondSignal(&fset->canCommit);
+    }
  }
-  taosThreadMutexUnlock(&fs->commitMutex);
  return 0;
 }

-int32_t tsdbFSCheckCommit(STFileSystem *fs) {
-  taosThreadMutexLock(&fs->commitMutex);
-  while (fs->blockCommit) {
-    taosThreadCondWait(&fs->canCommit, &fs->commitMutex);
+int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) {
+  taosThreadMutexLock(&tsdb->mutex);
+  STFileSet *fset;
+  tsdbFSGetFSet(tsdb->pFS, fid, &fset);
+  if (fset) {
+    while (fset->blockCommit) {
+      fset->numWaitCommit++;
+      taosThreadCondWait(&fset->canCommit, &tsdb->mutex);
+      fset->numWaitCommit--;
    }
-  taosThreadMutexUnlock(&fs->commitMutex);
+  }
+  taosThreadMutexUnlock(&tsdb->mutex);
  return 0;
 }

+// IMPORTANT: the caller must hold fs->tsdb->mutex
 int32_t tsdbFSEditCommit(STFileSystem *fs) {
  int32_t code = 0;
  int32_t lino = 0;
@ -867,36 +863,57 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
  TSDB_CHECK_CODE(code, lino, _exit);

  // schedule merge
-  if (fs->tsdb->pVnode->config.sttTrigger > 1) {
-    STFileSet *fset;
  int32_t sttTrigger = fs->tsdb->pVnode->config.sttTrigger;
-    bool       schedMerge = false;
-    bool       blockCommit = false;
-
+  if (sttTrigger > 1) {
+    STFileSet *fset;
    TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) {
-      if (TARRAY2_SIZE(fset->lvlArr) == 0) continue;
+      if (TARRAY2_SIZE(fset->lvlArr) == 0) {
+        tsdbFSSetBlockCommit(fset, false);
+        continue;
+      }

      SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr);
-      if (lvl->level != 0) continue;
+      if (lvl->level != 0) {
+        tsdbFSSetBlockCommit(fset, false);
+        continue;
+      }

      int32_t numFile = TARRAY2_SIZE(lvl->fobjArr);
      if (numFile >= sttTrigger) {
-        schedMerge = true;
-      }
-
-      if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) {
-        blockCommit = true;
-      }
-
-      if (schedMerge && blockCommit) break;
-    }
-
-    if (schedMerge) {
-      code = tsdbFSScheduleBgTask(fs, TSDB_BG_TASK_MERGER, tsdbMerge, NULL, fs->tsdb, NULL);
+        // launch merge
+        code = tsdbSchedMerge(fs->tsdb, fset->fid);
        TSDB_CHECK_CODE(code, lino, _exit);
      }

-    tsdbFSSetBlockCommit(fs, blockCommit);
+      if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) {
+        tsdbFSSetBlockCommit(fset, true);
+      } else {
+        tsdbFSSetBlockCommit(fset, false);
+      }
+    }
+  }
+
+  // clear empty level and fset
+  int32_t i = 0;
+  while (i < TARRAY2_SIZE(fs->fSetArr)) {
+    STFileSet *fset = TARRAY2_GET(fs->fSetArr, i);
+
+    int32_t j = 0;
+    while (j < TARRAY2_SIZE(fset->lvlArr)) {
+      SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j);
+
+      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
+        TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear);
+      } else {
+        j++;
+      }
+    }
+
+    if (tsdbTFileSetIsEmpty(fset) && fset->bgTaskRunning == NULL) {
+      TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear);
+    } else {
+      i++;
+    }
  }

 _exit:
@ -933,15 +950,15 @@ int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {

  TARRAY2_INIT(fsetArr[0]);

-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
  TARRAY2_FOREACH(fs->fSetArr, fset) {
-    code = tsdbTFileSetInitDup(fs->tsdb, fset, &fset1);
+    code = tsdbTFileSetInitCopy(fs->tsdb, fset, &fset1);
    if (code) break;

    code = TARRAY2_APPEND(fsetArr[0], fset1);
    if (code) break;
  }
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);

  if (code) {
    TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear);
@ -961,9 +978,9 @@ int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) {
 }

 int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
  int32_t code = tsdbFSCreateRefSnapshotWithoutLock(fs, fsetArr);
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
  return code;
 }

@ -1017,7 +1034,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
    }
  }

-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
  TARRAY2_FOREACH(fs->fSetArr, fset) {
    int64_t ever = VERSION_MAX;
    if (pHash) {
@ -1034,7 +1051,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
    code = TARRAY2_APPEND(fsetArr[0], fset1);
    if (code) break;
  }
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);

 _out:
  if (code) {
@ -1089,7 +1106,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
    }
  }

-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
  TARRAY2_FOREACH(fs->fSetArr, fset) {
    int64_t sver1 = sver;
    int64_t ever1 = ever;
@ -1118,7 +1135,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev

    fsr1 = NULL;
  }
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);

  if (code) {
    tsdbTSnapRangeClear(&fsr1);
@ -1137,59 +1154,69 @@ _out:
 const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"};

 static int32_t tsdbFSRunBgTask(void *arg) {
-  STFileSystem *fs = (STFileSystem *)arg;
+  STFSBgTask   *task = (STFSBgTask *)arg;
+  STFileSystem *fs = task->fs;
+  STFileSet    *fset;

-  ASSERT(fs->bgTaskRunning != NULL);
+  tsdbFSGetFSet(fs, task->fid, &fset);

-  fs->bgTaskRunning->launchTime = taosGetTimestampMs();
-  fs->bgTaskRunning->run(fs->bgTaskRunning->arg);
-  fs->bgTaskRunning->finishTime = taosGetTimestampMs();
+  ASSERT(fset != NULL && fset->bgTaskRunning == task);
+
+  task->launchTime = taosGetTimestampMs();
+  task->run(task->arg);
+  task->finishTime = taosGetTimestampMs();

  tsdbDebug("vgId:%d bg task:%s task id:%" PRId64 " finished, schedule time:%" PRId64 " launch time:%" PRId64
            " finish time:%" PRId64,
-            TD_VID(fs->tsdb->pVnode), gFSBgTaskName[fs->bgTaskRunning->type], fs->bgTaskRunning->taskid,
-            fs->bgTaskRunning->scheduleTime, fs->bgTaskRunning->launchTime, fs->bgTaskRunning->finishTime);
+            TD_VID(fs->tsdb->pVnode), gFSBgTaskName[task->type], task->taskid, task->scheduleTime, task->launchTime,
+            task->finishTime);

-  taosThreadMutexLock(fs->mutex);
+  taosThreadMutexLock(&fs->tsdb->mutex);

  // free last
-  tsdbDoDoneBgTask(fs, fs->bgTaskRunning);
-  fs->bgTaskRunning = NULL;
+  tsdbDoDoneBgTask(fs, task);
+  fset->bgTaskRunning = NULL;

  // schedule next
-  if (fs->bgTaskNum > 0) {
+  if (fset->bgTaskNum > 0) {
    if (fs->stop) {
-      while (fs->bgTaskNum > 0) {
-        STFSBgTask *task = fs->bgTaskQueue->next;
-        task->prev->next = task->next;
-        task->next->prev = task->prev;
-        fs->bgTaskNum--;
-        tsdbDoDoneBgTask(fs, task);
+      while (fset->bgTaskNum > 0) {
+        STFSBgTask *nextTask = fset->bgTaskQueue->next;
+        nextTask->prev->next = nextTask->next;
+        nextTask->next->prev = nextTask->prev;
+        fset->bgTaskNum--;
+        tsdbDoDoneBgTask(fs, nextTask);
      }
    } else {
      // pop task from head
-      fs->bgTaskRunning = fs->bgTaskQueue->next;
-      fs->bgTaskRunning->prev->next = fs->bgTaskRunning->next;
-      fs->bgTaskRunning->next->prev = fs->bgTaskRunning->prev;
-      fs->bgTaskNum--;
-      vnodeScheduleTaskEx(1, tsdbFSRunBgTask, arg);
+      fset->bgTaskRunning = fset->bgTaskQueue->next;
+      fset->bgTaskRunning->prev->next = fset->bgTaskRunning->next;
+      fset->bgTaskRunning->next->prev = fset->bgTaskRunning->prev;
+      fset->bgTaskNum--;
+      vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fset->bgTaskRunning);
    }
  }

-  taosThreadMutexUnlock(fs->mutex);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
  return 0;
 }

-static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int32_t (*run)(void *),
+// IMPORTANT: the caller must hold the fs->tsdb->mutex
+int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *),
                             void (*destroy)(void *), void *arg, int64_t *taskid) {
  if (fs->stop) {
    if (destroy) {
      destroy(arg);
    }
-    return 0;  // TODO: use a better error code
+    return 0;
  }

-  for (STFSBgTask *task = fs->bgTaskQueue->next; task != fs->bgTaskQueue; task = task->next) {
+  STFileSet *fset;
+  tsdbFSGetFSet(fs, fid, &fset);
+
+  ASSERT(fset != NULL);
+
+  for (STFSBgTask *task = fset->bgTaskQueue->next; task != fset->bgTaskQueue; task = task->next) {
    if (task->type == type) {
      if (destroy) {
        destroy(arg);
@ -1203,22 +1230,24 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int
  if (task == NULL) return TSDB_CODE_OUT_OF_MEMORY;
  taosThreadCondInit(task->done, NULL);

+  task->fs = fs;
+  task->fid = fid;
  task->type = type;
  task->run = run;
-  task->free = destroy;
+  task->destroy = destroy;
  task->arg = arg;
  task->scheduleTime = taosGetTimestampMs();
  task->taskid = ++fs->taskid;

-  if (fs->bgTaskRunning == NULL && fs->bgTaskNum == 0) {
+  if (fset->bgTaskRunning == NULL && fset->bgTaskNum == 0) {
    // launch task directly
-    fs->bgTaskRunning = task;
-    vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fs);
+    fset->bgTaskRunning = task;
+    vnodeScheduleTaskEx(1, tsdbFSRunBgTask, task);
  } else {
    // add to the queue tail
-    fs->bgTaskNum++;
-    task->next = fs->bgTaskQueue;
-    task->prev = fs->bgTaskQueue->prev;
+    fset->bgTaskNum++;
+    task->next = fset->bgTaskQueue;
+    task->prev = fset->bgTaskQueue->prev;
    task->prev->next = task;
    task->next->prev = task;
  }
@ -1227,68 +1256,30 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int
  return 0;
 }

-int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg,
-                             int64_t *taskid) {
-  taosThreadMutexLock(fs->mutex);
-  int32_t code = tsdbFSScheduleBgTaskImpl(fs, type, run, free, arg, taskid);
-  taosThreadMutexUnlock(fs->mutex);
-  return code;
-}
+int32_t tsdbFSDisableBgTask(STFileSystem *fs) {
+  taosThreadMutexLock(&fs->tsdb->mutex);
+  for (;;) {
+    fs->stop = true;
+    bool done = true;

-int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid) {
-  STFSBgTask *task = NULL;
-
-  taosThreadMutexLock(fs->mutex);
-
-  if (fs->bgTaskRunning && fs->bgTaskRunning->taskid == taskid) {
-    task = fs->bgTaskRunning;
-  } else {
-    for (STFSBgTask *taskt = fs->bgTaskQueue->next; taskt != fs->bgTaskQueue; taskt = taskt->next) {
-      if (taskt->taskid == taskid) {
-        task = taskt;
+    STFileSet *fset;
+    TARRAY2_FOREACH(fs->fSetArr, fset) {
+      if (fset->bgTaskRunning) {
+        tsdbDoWaitBgTask(fs, fset->bgTaskRunning);
+        done = false;
        break;
      }
    }
-  }

-  if (task) {
-    tsdbDoWaitBgTask(fs, task);
+    if (done) break;
  }
-
-  taosThreadMutexUnlock(fs->mutex);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
  return 0;
 }

-int32_t tsdbFSWaitAllBgTask(STFileSystem *fs) {
-  taosThreadMutexLock(fs->mutex);
-
-  while (fs->bgTaskRunning) {
-    taosThreadCondWait(fs->bgTaskRunning->done, fs->mutex);
-  }
-
-  taosThreadMutexUnlock(fs->mutex);
-  return 0;
-}
-
-static int32_t tsdbFSDoDisableBgTask(STFileSystem *fs) {
-  fs->stop = true;
-
-  if (fs->bgTaskRunning) {
-    tsdbDoWaitBgTask(fs, fs->bgTaskRunning);
-  }
-  return 0;
-}
-
-int32_t tsdbFSDisableBgTask(STFileSystem *fs) {
-  taosThreadMutexLock(fs->mutex);
-  int32_t code = tsdbFSDoDisableBgTask(fs);
-  taosThreadMutexUnlock(fs->mutex);
-  return code;
-}
-
 int32_t tsdbFSEnableBgTask(STFileSystem *fs) {
-  taosThreadMutexLock(fs->mutex);
+  taosThreadMutexLock(&fs->tsdb->mutex);
  fs->stop = false;
-  taosThreadMutexUnlock(fs->mutex);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
  return 0;
 }
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.h
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h
@ -22,22 +22,11 @@
 extern "C" {
 #endif

-/* Exposed Handle */
-typedef struct STFileSystem STFileSystem;
-typedef struct STFSBgTask   STFSBgTask;
-// typedef TARRAY2(STFileSet *) TFileSetArray;
-
 typedef enum {
  TSDB_FEDIT_COMMIT = 1,  //
  TSDB_FEDIT_MERGE
 } EFEditT;

-typedef enum {
-  TSDB_BG_TASK_MERGER = 1,
-  TSDB_BG_TASK_RETENTION,
-  TSDB_BG_TASK_COMPACT,
-} EFSBgTaskT;
-
 typedef enum {
  TSDB_FCURRENT = 1,
  TSDB_FCURRENT_C,  // for commit
@ -67,37 +56,17 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e
 int32_t tsdbFSEditCommit(STFileSystem *fs);
 int32_t tsdbFSEditAbort(STFileSystem *fs);
 // background task
-int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg,
-                             int64_t *taskid);
-int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid);
-int32_t tsdbFSWaitAllBgTask(STFileSystem *fs);
+int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *),
+                             void (*destroy)(void *), void *arg, int64_t *taskid);
 int32_t tsdbFSDisableBgTask(STFileSystem *fs);
 int32_t tsdbFSEnableBgTask(STFileSystem *fs);
 // other
 int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset);
-int32_t tsdbFSCheckCommit(STFileSystem *fs);
+int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid);
 // utils
 int32_t save_fs(const TFileSetArray *arr, const char *fname);
 int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);

-struct STFSBgTask {
-  EFSBgTaskT type;
-  int32_t (*run)(void *arg);
-  void (*free)(void *arg);
-  void *arg;
-
-  TdThreadCond done[1];
-  int32_t      numWait;
-
-  int64_t taskid;
-  int64_t scheduleTime;
-  int64_t launchTime;
-  int64_t finishTime;
-
-  struct STFSBgTask *prev;
-  struct STFSBgTask *next;
-};
-
 /* Exposed Structs */
 struct STFileSystem {
  STsdb        *tsdb;
@ -109,17 +78,8 @@ struct STFileSystem {
  TFileSetArray fSetArrTmp[1];

  // background task queue
-  TdThreadMutex mutex[1];
  bool    stop;
  int64_t taskid;
-  int32_t       bgTaskNum;
-  STFSBgTask    bgTaskQueue[1];
-  STFSBgTask   *bgTaskRunning;
-
-  // block commit variables
-  TdThreadMutex commitMutex;
-  TdThreadCond  canCommit;
-  bool          blockCommit;
 };

 #ifdef __cplusplus
--- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c
@ -342,11 +342,6 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) {
      int32_t    idx = TARRAY2_SEARCH_IDX(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ);
      ASSERT(idx >= 0);
      TARRAY2_REMOVE(lvl->fobjArr, idx, tsdbSttLvlClearFObj);
-
-      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
-        // TODO: remove the stt level if no file exists anymore
-        // TARRAY2_REMOVE(&fset->lvlArr, lvl - fset->lvlArr.data, tsdbSttLvlClear);
-      }
    } else {
      ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f));
      tsdbTFileObjUnref(fset->farr[op->of.type]);
@ -454,10 +449,22 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
  fset[0]->fid = fid;
  fset[0]->maxVerValid = VERSION_MAX;
  TARRAY2_INIT(fset[0]->lvlArr);
+
+  // background task queue
+  fset[0]->bgTaskNum = 0;
+  fset[0]->bgTaskQueue->next = fset[0]->bgTaskQueue;
+  fset[0]->bgTaskQueue->prev = fset[0]->bgTaskQueue;
+  fset[0]->bgTaskRunning = NULL;
+
+  // block commit variables
+  taosThreadCondInit(&fset[0]->canCommit, NULL);
+  fset[0]->numWaitCommit = 0;
+  fset[0]->blockCommit = false;
+
  return 0;
 }

-int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) {
+int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) {
  int32_t code = tsdbTFileSetInit(fset1->fid, fset);
  if (code) return code;

@ -588,21 +595,23 @@ int32_t tsdbTFileSetClear(STFileSet **fset) {

  TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear);

+  taosThreadCondDestroy(&fset[0]->canCommit);
  taosMemoryFree(fset[0]);
  fset[0] = NULL;

  return 0;
 }

-int32_t tsdbTFileSetRemove(STFileSet **fset) {
+int32_t tsdbTFileSetRemove(STFileSet *fset) {
+  if (fset == NULL) return 0;
+
  for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
-    if (fset[0]->farr[ftype] == NULL) continue;
-    tsdbTFileObjRemove(fset[0]->farr[ftype]);
+    if (fset->farr[ftype] == NULL) continue;
+    tsdbTFileObjRemove(fset->farr[ftype]);
  }

-  TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlRemove);
-  taosMemoryFree(fset[0]);
-  fset[0] = NULL;
+  TARRAY2_DESTROY(fset->lvlArr, tsdbSttLvlRemove);
+
  return 0;
 }

--- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h
+++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h
@ -28,6 +28,8 @@ typedef struct SSttLvl   SSttLvl;
 typedef TARRAY2(STFileObj *) TFileObjArray;
 typedef TARRAY2(SSttLvl *) TSttLvlArray;
 typedef TARRAY2(STFileOp) TFileOpArray;
+typedef struct STFileSystem STFileSystem;
+typedef struct STFSBgTask   STFSBgTask;

 typedef enum {
  TSDB_FOP_NONE = 0,
@ -41,10 +43,10 @@ typedef enum {

 // init/clear
 int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset);
-int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
+int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
 int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
 int32_t tsdbTFileSetClear(STFileSet **fset);
-int32_t tsdbTFileSetRemove(STFileSet **fset);
+int32_t tsdbTFileSetRemove(STFileSet *fset);

 int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
                                    TFileOpArray *fopArr);
@ -58,6 +60,7 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset);
 // cmpr
 int32_t tsdbTFileSetCmprFn(const STFileSet **fset1, const STFileSet **fset2);
 // edit
+int32_t tsdbSttLvlClear(SSttLvl **lvl);
 int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op);
 int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset);
 // max commit id
@ -70,6 +73,33 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset);
 int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl);
 int32_t tsdbSttLvlClear(SSttLvl **lvl);

+typedef enum {
+  TSDB_BG_TASK_MERGER = 1,
+  TSDB_BG_TASK_RETENTION,
+  TSDB_BG_TASK_COMPACT,
+} EFSBgTaskT;
+
+struct STFSBgTask {
+  STFileSystem *fs;
+  int32_t       fid;
+
+  EFSBgTaskT type;
+  int32_t (*run)(void *arg);
+  void (*destroy)(void *arg);
+  void *arg;
+
+  TdThreadCond done[1];
+  int32_t      numWait;
+
+  int64_t taskid;
+  int64_t scheduleTime;
+  int64_t launchTime;
+  int64_t finishTime;
+
+  struct STFSBgTask *prev;
+  struct STFSBgTask *next;
+};
+
 struct STFileOp {
  tsdb_fop_t optype;
  int32_t    fid;
@ -87,6 +117,16 @@ struct STFileSet {
  int64_t      maxVerValid;
  STFileObj   *farr[TSDB_FTYPE_MAX];  // file array
  TSttLvlArray lvlArr[1];             // level array
+
+  // background task queue
+  int32_t     bgTaskNum;
+  STFSBgTask  bgTaskQueue[1];
+  STFSBgTask *bgTaskRunning;
+
+  // block commit variables
+  TdThreadCond canCommit;
+  int32_t      numWaitCommit;
+  bool         blockCommit;
 };

 struct STSnapRange {
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@ -191,7 +191,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid

  pMemTable->nDel++;
  pMemTable->minVer = TMIN(pMemTable->minVer, version);
-  pMemTable->maxVer = TMIN(pMemTable->maxVer, version);
+  pMemTable->maxVer = TMAX(pMemTable->maxVer, version);
  /*
  if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) {
    tsdbCacheDeleteLastrow(pTsdb->lruCache, pTbData->uid, eKey);
--- a/source/dnode/vnode/src/tsdb/tsdbMerge.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c
@ -15,11 +15,17 @@

 #include "tsdbMerge.h"

-#define TSDB_MAX_LEVEL 6  // means max level is 7
+#define TSDB_MAX_LEVEL 2  // means max level is 3

 typedef struct {
  STsdb  *tsdb;
-  TFileSetArray *fsetArr;
+  int32_t fid;
+} SMergeArg;
+
+typedef struct {
+  STsdb     *tsdb;
+  int32_t    fid;
+  STFileSet *fset;

  int32_t sttTrigger;
  int32_t maxRow;
@ -313,7 +319,6 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) {
      if (merger->ctx->fset->farr[ftype]) {
        config.files[ftype].exist = true;
        config.files[ftype].file = merger->ctx->fset->farr[ftype]->f[0];
-
      } else {
        config.files[ftype].exist = false;
      }
@ -397,13 +402,13 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
  code = tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE);
  TSDB_CHECK_CODE(code, lino, _exit);

-  taosThreadRwlockWrlock(&merger->tsdb->rwLock);
+  taosThreadMutexLock(&merger->tsdb->mutex);
  code = tsdbFSEditCommit(merger->tsdb->pFS);
  if (code) {
-    taosThreadRwlockUnlock(&merger->tsdb->rwLock);
+    taosThreadMutexUnlock(&merger->tsdb->mutex);
    TSDB_CHECK_CODE(code, lino, _exit);
  }
-  taosThreadRwlockUnlock(&merger->tsdb->rwLock);
+  taosThreadMutexUnlock(&merger->tsdb->mutex);

 _exit:
  if (code) {
@ -480,28 +485,19 @@ _exit:
 static int32_t tsdbDoMerge(SMerger *merger) {
  int32_t  code = 0;
  int32_t  lino = 0;
+  SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr);

-  STFileSet *fset;
-  TARRAY2_FOREACH(merger->fsetArr, fset) {
-    if (TARRAY2_SIZE(fset->lvlArr) == 0) continue;
+  if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0;
+  if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) return 0;

-    SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr);
-
-    if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) continue;
-
-    if (!merger->ctx->opened) {
  code = tsdbMergerOpen(merger);
  TSDB_CHECK_CODE(code, lino, _exit);
-    }

-    code = tsdbMergeFileSet(merger, fset);
+  code = tsdbMergeFileSet(merger, merger->fset);
  TSDB_CHECK_CODE(code, lino, _exit);
-  }

-  if (merger->ctx->opened) {
  code = tsdbMergerClose(merger);
  TSDB_CHECK_CODE(code, lino, _exit);
-  }

 _exit:
  if (code) {
@ -512,36 +508,73 @@ _exit:
  return code;
 }

-int32_t tsdbMerge(void *arg) {
-  int32_t code = 0;
-  int32_t lino = 0;
-  STsdb  *tsdb = (STsdb *)arg;
+static int32_t tsdbMergeGetFSet(SMerger *merger) {
+  STFileSet *fset;

-  SMerger merger[1] = {{
-      .tsdb = tsdb,
-      .sttTrigger = tsdb->pVnode->config.sttTrigger,
-  }};
-
-  if (merger->sttTrigger <= 1) {
+  taosThreadMutexLock(&merger->tsdb->mutex);
+  tsdbFSGetFSet(merger->tsdb->pFS, merger->fid, &fset);
+  if (fset == NULL) {
+    taosThreadMutexUnlock(&merger->tsdb->mutex);
    return 0;
  }

-  code = tsdbFSCreateCopySnapshot(tsdb->pFS, &merger->fsetArr);
+  int32_t code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset);
+  if (code) {
+    taosThreadMutexUnlock(&merger->tsdb->mutex);
+    return code;
+  }
+  taosThreadMutexUnlock(&merger->tsdb->mutex);
+  return 0;
+}
+
+static int32_t tsdbMerge(void *arg) {
+  int32_t    code = 0;
+  int32_t    lino = 0;
+  SMergeArg *mergeArg = (SMergeArg *)arg;
+  STsdb     *tsdb = mergeArg->tsdb;
+
+  SMerger merger[1] = {{
+      .tsdb = tsdb,
+      .fid = mergeArg->fid,
+      .sttTrigger = tsdb->pVnode->config.sttTrigger,
+  }};
+
+  if (merger->sttTrigger <= 1) return 0;
+
+  // copy snapshot
+  code = tsdbMergeGetFSet(merger);
  TSDB_CHECK_CODE(code, lino, _exit);

+  if (merger->fset == NULL) return 0;
+
+  // do merge
+  tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
  code = tsdbDoMerge(merger);
+  tsdbDebug("vgId:%d merge done, fid:%d", TD_VID(tsdb->pVnode), mergeArg->fid);
  TSDB_CHECK_CODE(code, lino, _exit);

-  tsdbFSDestroyCopySnapshot(&merger->fsetArr);
-
 _exit:
  if (code) {
    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
    tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code);
    taosMsleep(100);
    exit(EXIT_FAILURE);
-  } else if (merger->ctx->opened) {
-    tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
  }
+  tsdbTFileSetClear(&merger->fset);
+  return code;
+}
+
+int32_t tsdbSchedMerge(STsdb *tsdb, int32_t fid) {
+  SMergeArg *arg = taosMemoryMalloc(sizeof(*arg));
+  if (arg == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  arg->tsdb = tsdb;
+  arg->fid = fid;
+
+  int32_t code = tsdbFSScheduleBgTask(tsdb->pFS, fid, TSDB_BG_TASK_MERGER, tsdbMerge, taosMemoryFree, arg, NULL);
+  if (code) taosMemoryFree(arg);
+
  return code;
 }
--- a/source/dnode/vnode/src/tsdb/tsdbOpen.c
+++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c
@ -53,7 +53,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
  snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir);
  // taosRealPath(pTsdb->path, NULL, slen);
  pTsdb->pVnode = pVnode;
-  taosThreadRwlockInit(&pTsdb->rwLock, NULL);
+  taosThreadMutexInit(&pTsdb->mutex, NULL);
  if (!pKeepCfg) {
    tsdbSetKeepCfg(pTsdb, &pVnode->config.tsdbCfg);
  } else {
@ -99,15 +99,14 @@ int tsdbClose(STsdb **pTsdb) {
    tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path,
              pdb->keepCfg.days, pdb->keepCfg.keep0, pdb->keepCfg.keep1, pdb->keepCfg.keep2,
              pdb->keepCfg.keepTimeOffset);
-    taosThreadRwlockWrlock(&(*pTsdb)->rwLock);
+    taosThreadMutexLock(&(*pTsdb)->mutex);
    tsdbMemTableDestroy((*pTsdb)->mem, true);
    (*pTsdb)->mem = NULL;
-    taosThreadRwlockUnlock(&(*pTsdb)->rwLock);
-
-    taosThreadRwlockDestroy(&(*pTsdb)->rwLock);
+    taosThreadMutexUnlock(&(*pTsdb)->mutex);

    tsdbCloseFS(&(*pTsdb)->pFS);
    tsdbCloseCache(*pTsdb);
+    taosThreadMutexDestroy(&(*pTsdb)->mutex);
    taosMemoryFreeClear(*pTsdb);
  }
  return 0;
--- a/source/dnode/vnode/src/tsdb/tsdbRead2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c
@ -1105,8 +1105,9 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p
         (pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer);
 }

-static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo,
-                                        int32_t* nextIndex, int32_t order, SBrinRecord* pRecord) {
+static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo,
+                                        STableBlockScanInfo* pTableBlockScanInfo, int32_t* nextIndex, int32_t order,
+                                        SBrinRecord* pRecord) {
  bool asc = ASCENDING_TRAVERSE(order);
  if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockIdxList) - 1) {
    return false;
@ -1119,7 +1120,8 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo
  int32_t step = asc ? 1 : -1;
  //  *nextIndex = pBlockInfo->tbBlockIdx + step;
  //  *pBlockIndex = *(SBlockIndex*)taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex);
-  STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step);
+  STableDataBlockIdx* pTableDataBlockIdx =
+      taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step);
  SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex);
  memcpy(pRecord, &p->record, sizeof(SBrinRecord));

@ -1145,7 +1147,8 @@ static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlock
  return -1;
 }

-static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, int32_t step) {
+static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index,
+                                             int32_t step) {
  if (index < 0 || index >= pBlockIter->numOfBlocks) {
    return -1;
  }
@ -1158,7 +1161,8 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
      for (int32_t i = index - 1; i >= pBlockIter->index; --i) {
        SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i);

-        STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
+        STableBlockScanInfo* pBlockScanInfo =
+            getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
        STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);
        pTableDataBlockIdx->globalIndex = i + 1;

@ -1168,13 +1172,13 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
      for (int32_t i = index + 1; i <= pBlockIter->index; ++i) {
        SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i);

-        STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
+        STableBlockScanInfo* pBlockScanInfo =
+            getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
        STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);
        pTableDataBlockIdx->globalIndex = i - 1;

        taosArraySet(pBlockIter->blockList, i - 1, pBlockInfo);
      }
-
    }

    taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock);
@ -1196,11 +1200,28 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
  }
 }

-static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SFileDataBlockInfo* pBlock) {
+static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, SLastBlockReader* pLastBlockReader, int32_t order) {
  bool ascScan = ASCENDING_TRAVERSE(order);
+  bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);

-  return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->record.firstKey)) ||
-         (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->record.lastKey));
+  int64_t key = 0;
+  if (bHasDataInLastBlock) {
+    int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+    key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt);
+  } else {
+    key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey;
+  }
+
+  return key;
+}
+
+static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock,
+                                     SLastBlockReader* pLastBlockReader, int32_t order) {
+  bool    ascScan = ASCENDING_TRAVERSE(order);
+  int64_t key = getBoarderKeyInFiles(pBlock, pLastBlockReader, order);
+
+  return (ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts < key)) ||
+         (!ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts > key));
 }

 static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) {
@ -1286,7 +1307,8 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo*
  int32_t     neighborIndex = 0;
  SBrinRecord rec = {0};

-  bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, pReader->info.order, &rec);
+  bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex,
+                                                 pReader->info.order, &rec);

  // overlap with neighbor
  if (hasNeighbor) {
@ -1420,9 +1442,7 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
  }
 }

-static void doPinSttBlock(SLastBlockReader* pLastBlockReader) {
-  tMergeTreePinSttBlock(&pLastBlockReader->mergeTree);
-}
+static void doPinSttBlock(SLastBlockReader* pLastBlockReader) { tMergeTreePinSttBlock(&pLastBlockReader->mergeTree); }

 static void doUnpinSttBlock(SLastBlockReader* pLastBlockReader) {
  tMergeTreeUnpinSttBlock(&pLastBlockReader->mergeTree);
@ -2219,8 +2239,9 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;

  TSDBROW *pRow = NULL, *piRow = NULL;
-  int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] :
-                                                                   (ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MAX : INT64_MIN);
+  int64_t  key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped))
+                     ? pBlockData->aTSKEY[pDumpInfo->rowIndex]
+                     : (ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MAX : INT64_MIN);
  if (pBlockScanInfo->iter.hasVal) {
    pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
  }
@ -2257,7 +2278,8 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock
  *loadNeighbor = false;

  SBrinRecord rec = {0};
-  bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pBlockScanInfo, &nextIndex, pReader->info.order, &rec);
+  bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pBlockScanInfo, &nextIndex,
+                                                 pReader->info.order, &rec);
  if (!hasNeighbor) {  // do nothing
    return code;
  }
@ -2662,6 +2684,15 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
  }
 }

+static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, SLastBlockReader* pLastBlockReader, bool asc) {
+  if(!hasDataInLastBlock(pLastBlockReader)) {
+    return true;
+  } else {
+    int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+    return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt);
+  }
+}
+
 static int32_t doBuildDataBlock(STsdbReader* pReader) {
  int32_t code = TSDB_CODE_SUCCESS;

@ -2697,17 +2728,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {

    // build composed data block
    code = buildComposedDataBlock(pReader);
-  } else if (bufferDataInFileBlockGap(pReader->info.order, keyInBuf, pBlockInfo)) {
-    // data in memory that are earlier than current file block
+  } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pLastBlockReader, pReader->info.order)) {
+    // data in memory that are earlier than current file block and stt blocks
    // rows in buffer should be less than the file block in asc, greater than file block in desc
-    int64_t endKey =
-        (ASCENDING_TRAVERSE(pReader->info.order)) ? pBlockInfo->record.firstKey : pBlockInfo->record.lastKey;
+    int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pLastBlockReader, pReader->info.order);
    code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
  } else {
-    bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);
-    int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN;
-    if (!bHasDataInLastBlock ||
-        ((asc && pBlockInfo->record.lastKey < tsLast) || (!asc && pBlockInfo->record.firstKey > tsLast))) {
+    if (notOverlapWithSttFiles(pBlockInfo, pLastBlockReader, asc)) {
      // whole block is required, return it directly
      SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
      pInfo->rows = pBlockInfo->record.numRow;
@ -2718,7 +2745,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
      setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);

      // update the last key for the corresponding table
-      pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? pInfo->window.ekey : pInfo->window.skey;
+      pScanInfo->lastKey = asc ? pInfo->window.ekey : pInfo->window.skey;
      tsdbDebug("%p uid:%" PRIu64
                " clean file block retrieved from file, global index:%d, "
                "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
@ -2745,10 +2772,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
        }

        // data in stt now overlaps with current active file data block, need to composed with file data block.
-        int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
-        if ((keyInStt >= pBlockInfo->record.firstKey && asc) || (keyInStt <= pBlockInfo->record.lastKey && (!asc))) {
-          tsdbDebug("%p keyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
-                    keyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
+        int64_t lastKeyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+        if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) ||
+            (lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) {
+          tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
+                    lastKeyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
          break;
        }
      }
@ -4907,12 +4935,12 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs
  SVersionRange* pRange = &pReader->info.verRange;

  // lock
-  taosThreadRwlockRdlock(&pTsdb->rwLock);
+  taosThreadMutexLock(&pTsdb->mutex);

  // alloc
  STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap));
  if (pSnap == NULL) {
-    taosThreadRwlockUnlock(&pTsdb->rwLock);
+    taosThreadMutexUnlock(&pTsdb->mutex);
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
  }
@ -4922,7 +4950,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs
    pSnap->pMem = pTsdb->mem;
    pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode));
    if (pSnap->pNode == NULL) {
-      taosThreadRwlockUnlock(&pTsdb->rwLock);
+      taosThreadMutexUnlock(&pTsdb->mutex);
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _exit;
    }
@ -4937,7 +4965,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs
    pSnap->pIMem = pTsdb->imem;
    pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode));
    if (pSnap->pINode == NULL) {
-      taosThreadRwlockUnlock(&pTsdb->rwLock);
+      taosThreadMutexUnlock(&pTsdb->mutex);
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _exit;
    }
@ -4952,7 +4980,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs
  code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray);

  // unlock
-  taosThreadRwlockUnlock(&pTsdb->rwLock);
+  taosThreadMutexUnlock(&pTsdb->mutex);

  if (code == TSDB_CODE_SUCCESS) {
    tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode));
@ -5005,4 +5033,5 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) {
  pReader->status.fileIter.pLastBlockReader->mergeTree.idStr = pReader->idStr;
 }

-void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ }
+void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/
+}
--- a/source/dnode/vnode/src/tsdb/tsdbRetention.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@ -25,11 +25,6 @@ typedef struct {

  TFileSetArray *fsetArr;
  TFileOpArray   fopArr[1];
-
-  struct {
-    int32_t    fsetArrIdx;
-    STFileSet *fset;
-  } ctx[1];
 } SRTNer;

 static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
@ -229,8 +224,8 @@ _exit:

 typedef struct {
  STsdb  *tsdb;
-  int32_t sync;
  int64_t now;
+  int32_t fid;
 } SRtnArg;

 static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) {
@ -265,15 +260,15 @@ static int32_t tsdbDoRetentionEnd(SRTNer *rtner) {
  code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE);
  TSDB_CHECK_CODE(code, lino, _exit);

-  taosThreadRwlockWrlock(&rtner->tsdb->rwLock);
+  taosThreadMutexLock(&rtner->tsdb->mutex);

  code = tsdbFSEditCommit(rtner->tsdb->pFS);
  if (code) {
-    taosThreadRwlockUnlock(&rtner->tsdb->rwLock);
+    taosThreadMutexUnlock(&rtner->tsdb->mutex);
    TSDB_CHECK_CODE(code, lino, _exit);
  }

-  taosThreadRwlockUnlock(&rtner->tsdb->rwLock);
+  taosThreadMutexUnlock(&rtner->tsdb->mutex);

  TARRAY2_DESTROY(rtner->fopArr, NULL);

@ -287,27 +282,18 @@ _exit:
  return code;
 }

-static int32_t tsdbDoRetention2(void *arg) {
+static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
  int32_t    code = 0;
  int32_t    lino = 0;
-  SRTNer  rtner[1] = {0};
+  STFileObj *fobj = NULL;
+  int32_t    expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);

-  code = tsdbDoRetentionBegin(arg, rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  for (rtner->ctx->fsetArrIdx = 0; rtner->ctx->fsetArrIdx < TARRAY2_SIZE(rtner->fsetArr); rtner->ctx->fsetArrIdx++) {
-    rtner->ctx->fset = TARRAY2_GET(rtner->fsetArr, rtner->ctx->fsetArrIdx);
-
-    STFileObj *fobj;
-    int32_t    expLevel = tsdbFidLevel(rtner->ctx->fset->fid, &rtner->tsdb->keepCfg, rtner->now);
-
-    if (expLevel < 0) {  // remove the file set
-      for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) {
+  if (expLevel < 0) {  // remove the fileset
+    for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) {
      if (fobj == NULL) continue;

      int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
-        if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1 &&
-            !taosCheckExistFile(fobj->fname)) {
+      if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
        code = tsdbRemoveFileObjectS3(rtner, fobj);
        TSDB_CHECK_CODE(code, lino, _exit);
      } else {
@ -317,15 +303,15 @@ static int32_t tsdbDoRetention2(void *arg) {
    }

    SSttLvl *lvl;
-      TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) {
+    TARRAY2_FOREACH(fset->lvlArr, lvl) {
      TARRAY2_FOREACH(lvl->fobjArr, fobj) {
        code = tsdbDoRemoveFileObject(rtner, fobj);
        TSDB_CHECK_CODE(code, lino, _exit);
      }
    }
-    } else if (expLevel == 0) {
-      continue;
-    } else {
+  } else if (expLevel == 0) {  // only migrate to upper level
+    return 0;
+  } else {  // migrate
    SDiskID did;

    if (tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did) < 0) {
@ -335,7 +321,7 @@ static int32_t tsdbDoRetention2(void *arg) {
    tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did);

    // data
-      for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) {
+    for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) {
      if (fobj == NULL) continue;

        int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
@ -373,10 +359,11 @@ static int32_t tsdbDoRetention2(void *arg) {
        TSDB_CHECK_CODE(code, lino, _exit);
        //}
      }
+    }

    // stt
    SSttLvl *lvl;
-      TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) {
+    TARRAY2_FOREACH(fset->lvlArr, lvl) {
      TARRAY2_FOREACH(lvl->fobjArr, fobj) {
        if (fobj->f->did.level == did.level) continue;

@ -385,10 +372,6 @@ static int32_t tsdbDoRetention2(void *arg) {
      }
    }
  }
-  }
-
-  code = tsdbDoRetentionEnd(rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);

 _exit:
  if (code) {
@ -405,30 +388,105 @@ _exit:
  return code;
 }

-static void tsdbFreeRtnArg(void *arg) {
-  SRtnArg *rArg = (SRtnArg *)arg;
-  if (rArg->sync) {
-    tsem_post(&rArg->tsdb->pVnode->canCommit);
+static int32_t tsdbDoRetentionSync(void *arg) {
+  int32_t code = 0;
+  int32_t lino = 0;
+  SRTNer  rtner[1] = {0};
+
+  code = tsdbDoRetentionBegin(arg, rtner);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  STFileSet *fset;
+  TARRAY2_FOREACH(rtner->fsetArr, fset) {
+    code = tsdbDoRetentionOnFileSet(rtner, fset);
+    TSDB_CHECK_CODE(code, lino, _exit);
  }
-  taosMemoryFree(arg);
+
+  code = tsdbDoRetentionEnd(rtner);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+  }
+  tsem_post(&((SRtnArg *)arg)->tsdb->pVnode->canCommit);
+  return code;
 }

-int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
-  SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
-  if (arg == NULL) return TSDB_CODE_OUT_OF_MEMORY;
-  arg->tsdb = tsdb;
-  arg->sync = sync;
-  arg->now = now;
+static int32_t tsdbDoRetentionAsync(void *arg) {
+  int32_t code = 0;
+  int32_t lino = 0;
+  SRTNer  rtner[1] = {0};

-  if (sync) {
-    tsem_wait(&tsdb->pVnode->canCommit);
+  code = tsdbDoRetentionBegin(arg, rtner);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  STFileSet *fset;
+  TARRAY2_FOREACH(rtner->fsetArr, fset) {
+    if (fset->fid != ((SRtnArg *)arg)->fid) continue;
+
+    code = tsdbDoRetentionOnFileSet(rtner, fset);
+    TSDB_CHECK_CODE(code, lino, _exit);
  }

-  int64_t taskid;
-  int32_t code =
-      tsdbFSScheduleBgTask(tsdb->pFS, TSDB_BG_TASK_RETENTION, tsdbDoRetention2, tsdbFreeRtnArg, arg, &taskid);
+  code = tsdbDoRetentionEnd(rtner);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
  if (code) {
-    tsdbFreeRtnArg(arg);
+    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
  }
  return code;
 }
+
+static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
+
+int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
+  int32_t code = 0;
+
+  if (sync) {  // sync retention
+    SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
+    if (arg == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    arg->tsdb = tsdb;
+    arg->now = now;
+    arg->fid = INT32_MAX;
+
+    tsem_wait(&tsdb->pVnode->canCommit);
+    code = vnodeScheduleTask(tsdbDoRetentionSync, arg);
+    if (code) {
+      tsem_post(&tsdb->pVnode->canCommit);
+      taosMemoryFree(arg);
+      return code;
+    }
+  } else {  // async retention
+    taosThreadMutexLock(&tsdb->mutex);
+
+    STFileSet *fset;
+    TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
+      SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
+      if (arg == NULL) {
+        taosThreadMutexUnlock(&tsdb->mutex);
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+
+      arg->tsdb = tsdb;
+      arg->now = now;
+      arg->fid = fset->fid;
+
+      code = tsdbFSScheduleBgTask(tsdb->pFS, fset->fid, TSDB_BG_TASK_RETENTION, tsdbDoRetentionAsync, tsdbFreeRtnArg,
+                                  arg, NULL);
+      if (code) {
+        tsdbFreeRtnArg(arg);
+        taosThreadMutexUnlock(&tsdb->mutex);
+        return code;
+      }
+    }
+
+    taosThreadMutexUnlock(&tsdb->mutex);
+  }
+
+  return code;
+}
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@ -1095,17 +1095,17 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) {
    code = tsdbFSEditAbort(writer[0]->tsdb->pFS);
    TSDB_CHECK_CODE(code, lino, _exit);
  } else {
-    taosThreadRwlockWrlock(&writer[0]->tsdb->rwLock);
+    taosThreadMutexLock(&writer[0]->tsdb->mutex);

    code = tsdbFSEditCommit(writer[0]->tsdb->pFS);
    if (code) {
-      taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock);
+      taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
      TSDB_CHECK_CODE(code, lino, _exit);
    }

    writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL;

-    taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock);
+    taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
  }
  tsdbFSEnableBgTask(tsdb->pFS);

@ -1299,7 +1299,7 @@ static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) {
  }

  int32_t code = 0;
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
  STFileSet* fset;
  TARRAY2_FOREACH(fs->fSetArr, fset) {
    STsdbSnapPartition* pItem = NULL;
@ -1311,7 +1311,7 @@ static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) {
    code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn);
    ASSERT(code == 0);
  }
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);

  if (code) {
    TARRAY2_DESTROY(pList, tsdbSnapPartitionClear);
@ -1521,7 +1521,8 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) {
    return pVnode->pTsdb->pFS->fsstate;
  }
  for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) {
-    if (SMA_RSMA_GET_TSDB(pVnode, lvl)->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
+    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl);
+    if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
      return TSDB_FS_STATE_INCOMPLETE;
    }
  }
--- a/source/dnode/vnode/src/vnd/vnodeCfg.c
+++ b/source/dnode/vnode/src/vnd/vnodeCfg.c
@ -106,7 +106,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
  if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
  if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
  if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
-  if (pCfg->tsdbCfg.retentions[0].freq > 0) {
+  if (pCfg->tsdbCfg.retentions[0].keep > 0) {
    int32_t nRetention = 1;
    if (pCfg->tsdbCfg.retentions[1].freq > 0) {
      ++nRetention;
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@ -584,6 +584,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *

  // commit json
  if (!rollback) {
+    ASSERT(pVnode->config.vgId == pWriter->info.config.vgId);
    pWriter->info.state.committed = pWriter->ever;
    pVnode->config = pWriter->info.config;
    pVnode->state = (SVState){.committed = pWriter->info.state.committed,
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@ -20,6 +20,8 @@
 #include "vndCos.h"
 #include "vnode.h"
 #include "vnodeInt.h"
+#include "audit.h"
+#include "tstrbuild.h"

 static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@ -585,7 +587,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
      }
    } break;
    case TDMT_VND_STREAM_TASK_RESET: {
-      if (pVnode->restored /* && vnodeIsLeader(pVnode)*/) {
+      if (pVnode->restored && vnodeIsLeader(pVnode)) {
        tqProcessTaskResetReq(pVnode->pTq, pMsg);
      }
    } break;
@ -887,6 +889,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
  char               tbName[TSDB_TABLE_FNAME_LEN];
  STbUidStore       *pStore = NULL;
  SArray            *tbUids = NULL;
+  SArray            *tbNames = NULL;

  pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
  pRsp->code = TSDB_CODE_SUCCESS;
@ -903,7 +906,8 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,

  rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp));
  tbUids = taosArrayInit(req.nReqs, sizeof(int64_t));
-  if (rsp.pArray == NULL || tbUids == NULL) {
+  tbNames = taosArrayInit(req.nReqs, sizeof(char*));
+  if (rsp.pArray == NULL || tbUids == NULL || tbNames == NULL) {
    rcode = -1;
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
@ -948,13 +952,10 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,

    taosArrayPush(rsp.pArray, &cRsp);

-    if (tsEnableAuditCreateTable) {
-      int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
-
-      SName name = {0};
-      tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);
-
-      auditRecord(NULL, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name));
+    if(tsEnableAuditCreateTable){
+      char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
+      strcpy(str, pCreateReq->name);
+      taosArrayPush(tbNames, &str);
    }
  }

@ -977,6 +978,30 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
  tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen);
  tEncodeSVCreateTbBatchRsp(&encoder, &rsp);

+  if(tsEnableAuditCreateTable){
+    int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
+
+    SName name = {0};
+    tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);
+
+    SStringBuilder sb = {0};
+    for(int32_t iReq = 0; iReq < req.nReqs; iReq++){
+      char** key = (char**)taosArrayGet(tbNames, iReq);
+      taosStringBuilderAppendStringLen(&sb, *key, strlen(*key));
+      if(iReq < req.nReqs - 1){
+        taosStringBuilderAppendChar(&sb, ',');
+      }
+      taosMemoryFreeClear(*key);
+    }
+
+    size_t    len = 0;
+    char*     keyJoined = taosStringBuilderGetResult(&sb, &len);
+
+    auditRecord(NULL, clusterId, "createTable", name.dbname, "", keyJoined, len);
+
+    taosStringBuilderDestroy(&sb);
+  }
+
 _exit:
  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
    pCreateReq = req.pReqs + iReq;
@ -988,6 +1013,7 @@ _exit:
  taosArrayDestroy(tbUids);
  tDecoderClear(&decoder);
  tEncoderClear(&encoder);
+  taosArrayDestroy(tbNames);
  return rcode;
 }

@ -1121,6 +1147,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in
  int32_t          ret;
  SArray          *tbUids = NULL;
  STbUidStore     *pStore = NULL;
+  SArray          *tbNames = NULL;

  pRsp->msgType = TDMT_VND_DROP_TABLE_RSP;
  pRsp->pCont = NULL;
@ -1139,7 +1166,8 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in
  // process req
  tbUids = taosArrayInit(req.nReqs, sizeof(int64_t));
  rsp.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbRsp));
-  if (tbUids == NULL || rsp.pArray == NULL) goto _exit;
+  tbNames = taosArrayInit(req.nReqs, sizeof(char*));
+  if (tbUids == NULL || rsp.pArray == NULL || tbNames == NULL) goto _exit;

  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
    SVDropTbReq *pDropTbReq = req.pReqs + iReq;
@ -1160,11 +1188,41 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in
    }

    taosArrayPush(rsp.pArray, &dropTbRsp);
+    
+    if(tsEnableAuditCreateTable){
+      char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
+      strcpy(str, pDropTbReq->name);
+      taosArrayPush(tbNames, &str);
+    }
  }

  tqUpdateTbUidList(pVnode->pTq, tbUids, false);
  tdUpdateTbUidList(pVnode->pSma, pStore, false);

+  if(tsEnableAuditCreateTable){
+    int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
+
+    SName name = {0};
+    tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);
+
+    SStringBuilder sb = {0};
+    for(int32_t iReq = 0; iReq < req.nReqs; iReq++){
+      char** key = (char**)taosArrayGet(tbNames, iReq);
+      taosStringBuilderAppendStringLen(&sb, *key, strlen(*key));
+      if(iReq < req.nReqs - 1){
+        taosStringBuilderAppendChar(&sb, ',');
+      }
+      taosMemoryFreeClear(*key);
+    }
+
+    size_t    len = 0;
+    char*     keyJoined = taosStringBuilderGetResult(&sb, &len);
+
+    auditRecord(NULL, clusterId, "dropTable", name.dbname, "", keyJoined, len);
+
+    taosStringBuilderDestroy(&sb);
+  }
+  
 _exit:
  taosArrayDestroy(tbUids);
  tdUidStoreFree(pStore);
@ -1175,6 +1233,7 @@ _exit:
  tEncodeSVDropTbBatchRsp(&encoder, &rsp);
  tEncoderClear(&encoder);
  taosArrayDestroy(rsp.pArray);
+  taosArrayDestroy(tbNames);
  return 0;
 }

--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@ -516,7 +516,10 @@ static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool
        pVnode->config.vgId, isApply, pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastConfigIndex);

  int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot);
-  vInfo("vgId:%d, apply vnode snapshot finished, code:0x%x", pVnode->config.vgId, code);
+  if (code != 0) {
+    vError("vgId:%d, failed to finish applying vnode snapshot since %s, code:0x%x", pVnode->config.vgId, terrstr(),
+           code);
+  }
  return code;
 }

@ -552,7 +555,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
  pVnode->restored = true;

  taosWLockLatch(&pVnode->pTq->pStreamMeta->lock);
-  if (pVnode->pTq->pStreamMeta->startInfo.startedAfterNodeUpdate) {
+  if (pVnode->pTq->pStreamMeta->startInfo.startAllTasksFlag) {
    vInfo("vgId:%d, sync restore finished, stream tasks will be launched by other thread", vgId);
    taosWUnLockLatch(&pVnode->pTq->pStreamMeta->lock);
    return;
@ -564,8 +567,8 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
      vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId);
    } else {
      vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId);
-      tqStartStreamTasks(pVnode->pTq);
-      tqCheckAndRunStreamTaskAsync(pVnode->pTq);
+      tqResetStreamTaskStatus(pVnode->pTq);
+      tqLaunchStreamTaskAsync(pVnode->pTq);
    }
  } else {
    vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId);
--- a/source/libs/command/src/command.c
+++ b/source/libs/command/src/command.c
@ -215,37 +215,28 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) {
  return v;
 }

-char* buildRetension(SArray* pRetension) {
+static char* buildRetension(SArray* pRetension) {
  size_t size = taosArrayGetSize(pRetension);
  if (size == 0) {
    return NULL;
  }

  char*   p1 = taosMemoryCalloc(1, 100);
-  SRetention* p = taosArrayGet(pRetension, 0);
-
  int32_t len = 0;

+  for (int32_t i = 0; i < size; ++i) {
+    SRetention* p = TARRAY_GET_ELEM(pRetension, i);
    int64_t     v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
    int64_t     v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
-  len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
-
-  if (size > 1) {
-    len += sprintf(p1 + len, ",");
-    p = taosArrayGet(pRetension, 1);
-
-    v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
-    v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
+    if (i == 0) {
+      len += sprintf(p1 + len, "-:%" PRId64 "%c", v2, p->keepUnit);
+    } else {
      len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
    }

-  if (size > 2) {
+    if (i < size - 1) {
      len += sprintf(p1 + len, ",");
-    p = taosArrayGet(pRetension, 2);
-
-    v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
-    v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
-    len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
+    }
  }

  return p1;
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@ -2083,6 +2083,34 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int
  return TSDB_CODE_SUCCESS;
 }

+static int32_t translateToTimestamp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (LIST_LENGTH(pFunc->pParameterList) != 2) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+  uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
+  if (!IS_STR_DATA_TYPE(para1Type) || !IS_STR_DATA_TYPE(para2Type)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+  pFunc->node.resType =
+      (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateToChar(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (LIST_LENGTH(pFunc->pParameterList) != 2) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+  uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
+  // currently only support to_char(timestamp, str)
+  if (!IS_STR_DATA_TYPE(para2Type) || !IS_TIMESTAMP_TYPE(para1Type)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+  pFunc->node.resType = (SDataType){.bytes = 4096, .type = TSDB_DATA_TYPE_VARCHAR};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
  if (2 != numOfParams && 3 != numOfParams) {
@ -3284,6 +3312,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
    .sprocessFunc = castFunction,
    .finalizeFunc = NULL
  },
+  {
+    .name = "to_timestamp",
+    .type = FUNCTION_TYPE_TO_TIMESTAMP,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateToTimestamp,
+    .getEnvFunc = NULL,
+    .initFunc = NULL,
+    .sprocessFunc = toTimestampFunction,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "to_char",
+    .type = FUNCTION_TYPE_TO_CHAR,
+    .classification = FUNC_MGT_SCALAR_FUNC,
+    .translateFunc = translateToChar,
+    .getEnvFunc = NULL,
+    .initFunc = NULL,
+    .sprocessFunc = toCharFunction,
+    .finalizeFunc = NULL
+  },
  {
    .name = "to_iso8601",
    .type = FUNCTION_TYPE_TO_ISO8601,
--- a/source/libs/parser/inc/sql.y
+++ b/source/libs/parser/inc/sql.y
@ -304,6 +304,7 @@ retention_list(A) ::= retention(B).
 retention_list(A) ::= retention_list(B) NK_COMMA retention(C).                    { A = addNodeToList(pCxt, B, C); }

 retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C).                          { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }
+retention(A) ::= NK_MINUS(B) NK_COLON NK_VARIABLE(C).                             { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }

 %type speed_opt                                                                   { int32_t }
 %destructor speed_opt                                                             { }
@ -787,6 +788,7 @@ table_alias(A) ::= NK_ID(B).
 %type column_alias                                                                { SToken }
 %destructor column_alias                                                          { }
 column_alias(A) ::= NK_ID(B).                                                     { A = B; }
+column_alias(A) ::= NK_ALIAS(B).                                                  { A = B; }

 %type user_name                                                                   { SToken }
 %destructor user_name                                                             { }
@ -873,6 +875,8 @@ expression_list(A) ::= expression_list(B) NK_COMMA expr_or_subquery(C).

 column_reference(A) ::= column_name(B).                                           { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
 column_reference(A) ::= table_name(B) NK_DOT column_name(C).                      { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
+column_reference(A) ::= NK_ALIAS(B).                                              { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
+column_reference(A) ::= table_name(B) NK_DOT NK_ALIAS(C).                         { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }

 pseudo_column(A) ::= ROWTS(B).                                                    { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
 pseudo_column(A) ::= TBNAME(B).                                                   { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@ -1030,8 +1030,12 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr

 SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { 
  if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
+    if (pCxt->pQueryCxt->biMode) {
+      ((SSelectStmt*)pStmt)->tagScan = true;
+    } else {  
      ((SSelectStmt*)pStmt)->tagScan = bSelectTags;
    }
+  }
  return pStmt;
 }

--- a/source/libs/parser/src/parTokenizer.c
+++ b/source/libs/parser/src/parTokenizer.c
@ -627,9 +627,21 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
    case 't':
    case 'F':
    case 'f': {
-      for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t)z[i]]; i++) {
+      bool hasNonAsciiChars = false;
+      for (i = 1;; i++) {
+        if ((z[i] & 0x80) != 0) {
+          // utf-8 characters
+          // currently, we support using utf-8 characters only in alias
+          hasNonAsciiChars = true;
+        } else if (isIdChar[(uint8_t)z[i]]) {
+        } else {
+          break;
+        }
+      }
+      if (hasNonAsciiChars) {
+        *tokenId = TK_NK_ALIAS; // must be alias
+        return i;
      }
-
      if ((i == 4 && strncasecmp(z, "true", 4) == 0) || (i == 5 && strncasecmp(z, "false", 5) == 0)) {
        *tokenId = TK_NK_BOOL;
        return i;
@ -638,10 +650,21 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
      return i;
    }
    default: {
-      if (((*z & 0x80) != 0) || !isIdChar[(uint8_t)*z]) {
+      if ((*z & 0x80) == 0 && !isIdChar[(uint8_t)*z]) {
        break;
      }
-      for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t)z[i]]; i++) {
+      bool hasNonAsciiChars = false;
+      for (i = 1; ; i++) {
+        if ((z[i] & 0x80) != 0) {
+          hasNonAsciiChars = true;
+        } else if (isIdChar[(uint8_t)z[i]]){
+        } else {
+          break;
+        }
+      }
+      if (hasNonAsciiChars) {
+        *tokenId = TK_NK_ALIAS;
+        return i;
      }
      *tokenId = tKeywordCode(z, i);
      return i;
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@ -3499,6 +3499,20 @@ static const char* getPrecisionStr(uint8_t precision) {
  return "unknown";
 }

+static int64_t getPrecisionMultiple(uint8_t precision) {
+  switch (precision) {
+    case TSDB_TIME_PRECISION_MILLI:
+      return 1;
+    case TSDB_TIME_PRECISION_MICRO:
+      return 1000;
+    case TSDB_TIME_PRECISION_NANO:
+      return 1000000;
+    default:
+      break;
+  }
+  return 1;
+}
+
 static void convertVarDuration(SValueNode* pOffset, uint8_t precision) {
  const int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
  const int8_t  units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND};
@ -3512,6 +3526,7 @@ static void convertVarDuration(SValueNode* pOffset, uint8_t precision) {
  pOffset->unit = units[precision];
 }

+static const int64_t  tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP;
 static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
  uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;

@ -3520,6 +3535,8 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
  if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) {
    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime,
                                getPrecisionStr(precision));
+  } else if (pInter->datum.i / getPrecisionMultiple(precision) > tsdbMaxKeepMS) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years");
  }

  if (NULL != pInterval->pOffset) {
@ -4576,7 +4593,7 @@ static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int
  return TSDB_CODE_SUCCESS;
 }

-static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions) {
+static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions, int8_t precision) {
  if (NULL == pRetentions) {
    return TSDB_CODE_SUCCESS;
  }
@ -4588,10 +4605,22 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
  SValueNode* pPrevFreq = NULL;
  SValueNode* pPrevKeep = NULL;
  SNode*      pRetention = NULL;
+  bool        firstFreq = true;
  FOREACH(pRetention, pRetentions) {
    SNode* pNode = NULL;
    FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) {
      SValueNode* pVal = (SValueNode*)pNode;
+      if (firstFreq) {
+        firstFreq = false;
+        if (pVal->literal[0] != '-' || strlen(pVal->literal) != 1) {
+          return generateSyntaxErrMsgExt(
+              &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+              "Invalid option retentions(freq): %s, the interval of 1st retention level should be '-'", pVal->literal);
+        }
+        pVal->unit = TIME_UNIT_SECOND;  // assign minimum unit
+        pVal->datum.i = 0;              // assign minimum value
+        continue;
+      }
      if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
        return pCxt->errCode;
      }
@ -4599,11 +4628,55 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete

    SValueNode* pFreq = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 0);
    SValueNode* pKeep = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 1);
-    if (pFreq->datum.i <= 0 || 'n' == pFreq->unit || 'y' == pFreq->unit || pFreq->datum.i >= pKeep->datum.i ||
-        (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) ||
-        (NULL != pPrevKeep && pPrevKeep->datum.i > pKeep->datum.i)) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option retentions");
+
+    ASSERTS(pFreq->isDuration && pKeep->isDuration, "Retentions freq/keep should have unit");
+
+    // check unit
+    if (pFreq->isDuration && TIME_UNIT_SECOND != pFreq->unit && TIME_UNIT_MINUTE != pFreq->unit &&
+        TIME_UNIT_HOUR != pFreq->unit && TIME_UNIT_DAY != pFreq->unit && TIME_UNIT_WEEK != pFreq->unit) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option retentions(freq): %s, only s, m, h, d, w allowed", pFreq->literal);
    }
+
+    if (pKeep->isDuration && TIME_UNIT_MINUTE != pKeep->unit && TIME_UNIT_HOUR != pKeep->unit &&
+        TIME_UNIT_DAY != pKeep->unit) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option retentions(keep): %s, only m, h, d allowed", pKeep->literal);
+    }
+
+    // check value range
+    if (pPrevFreq != NULL && pFreq->datum.i <= 0) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option retentions(freq): %s should larger than 0", pFreq->literal);
+    }
+    int64_t keepMinute = pKeep->datum.i / getUnitPerMinute(pKeep->node.resType.precision);
+    int64_t tsdbMaxKeep = TSDB_TIME_PRECISION_NANO == precision ? TSDB_MAX_KEEP_NS : TSDB_MAX_KEEP;
+    if (keepMinute < TSDB_MIN_KEEP || keepMinute > tsdbMaxKeep) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option retentions(keep): %" PRId64 "m, valid range: [%" PRIi64
+                                     "m, %" PRId64 "m]",
+                                     keepMinute, TSDB_MIN_KEEP, tsdbMaxKeep);
+    }
+
+    // check relationships
+    if (pFreq->datum.i >= pKeep->datum.i) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option retentions(freq/keep): %s should larger than %s", pKeep->literal,
+                                     pFreq->literal);
+    }
+    
+    if (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option retentions(freq): %s should larger than %s", pFreq->literal,
+                                     pPrevFreq->literal);
+    }
+
+    if (NULL != pPrevKeep && pPrevKeep->datum.i > pKeep->datum.i) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option retentions(keep): %s should not larger than %s",
+                                     pPrevKeep->literal, pKeep->literal);
+    }
+
    pPrevFreq = pFreq;
    pPrevKeep = pKeep;
  }
@ -4723,7 +4796,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
                             TSDB_DB_SINGLE_STABLE_OFF);
  }
  if (TSDB_CODE_SUCCESS == code) {
-    code = checkDbRetentionsOption(pCxt, pOptions->pRetentions);
+    code = checkDbRetentionsOption(pCxt, pOptions->pRetentions, pOptions->precision);
  }
  if (TSDB_CODE_SUCCESS == code) {
    code = checkDbEnumOption(pCxt, "schemaless", pOptions->schemaless, TSDB_DB_SCHEMALESS_ON, TSDB_DB_SCHEMALESS_OFF);
@ -5031,6 +5104,17 @@ static bool validRollupFunc(const char* pFunc) {
  return false;
 }

+static bool aggrRollupFunc(const char* pFunc) {
+  static const char*   aggrRollupFuncs[] = {"avg", "sum"};
+  static const int32_t numOfAggrRollupFuncs = (sizeof(aggrRollupFuncs) / sizeof(char*));
+  for (int i = 0; i < numOfAggrRollupFuncs; ++i) {
+    if (0 == strcmp(aggrRollupFuncs[i], pFunc)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs, bool createStable,
                                      SDbCfgInfo* pDbCfg) {
  if (NULL == pFuncs) {
@ -5104,7 +5188,8 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN
  return code;
 }

-static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols) {
+static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols,
+                                    SNodeList* pRollupFuncs) {
  int32_t ncols = LIST_LENGTH(pCols);
  if (ncols < TSDB_MIN_COLUMNS) {
    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
@ -5114,13 +5199,19 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in

  int32_t code = TSDB_CODE_SUCCESS;

-  bool    first = true;
+  int32_t colIndex = 0;
  int32_t rowSize = 0;
  SNode*  pNode = NULL;
+  char*   pFunc = NULL;
+  bool    isAggrRollup = false;
+
+  if (pRollupFuncs) {
+    pFunc = ((SFunctionNode*)nodesListGetNode(pRollupFuncs, 0))->functionName;
+    isAggrRollup = aggrRollupFunc(pFunc);
+  }
  FOREACH(pNode, pCols) {
    SColumnDefNode* pCol = (SColumnDefNode*)pNode;
-    if (first) {
-      first = false;
+    if (0 == colIndex) {
      if (TSDB_DATA_TYPE_TIMESTAMP != pCol->dataType.type) {
        code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN);
      }
@ -5140,6 +5231,15 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in
        code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
      }
    }
+
+    if (TSDB_CODE_SUCCESS == code && isAggrRollup && 0 != colIndex) {
+      if (pCol->dataType.type != TSDB_DATA_TYPE_FLOAT && pCol->dataType.type != TSDB_DATA_TYPE_DOUBLE) {
+        code =
+            generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN,
+                                    "Invalid column type: %s, only float/double allowed for %s", pCol->colName, pFunc);
+      }
+    }
+
    if (TSDB_CODE_SUCCESS == code) {
      code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES);
    }
@ -5148,6 +5248,8 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in
    } else {
      break;
    }
+    // next column
+    ++colIndex;
  }

  if (TSDB_CODE_SUCCESS == code && rowSize > TSDB_MAX_BYTES_PER_ROW) {
@ -5166,7 +5268,7 @@ static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt

  int32_t code = checkTableTagsSchema(pCxt, pHash, pStmt->pTags);
  if (TSDB_CODE_SUCCESS == code) {
-    code = checkTableColsSchema(pCxt, pHash, LIST_LENGTH(pStmt->pTags), pStmt->pCols);
+    code = checkTableColsSchema(pCxt, pHash, LIST_LENGTH(pStmt->pTags), pStmt->pCols, pStmt->pOptions->pRollupFuncs);
  }

  taosHashCleanup(pHash);
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@ -65,6 +65,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
      return "This statement is no longer supported";
    case TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL:
      return "Interval cannot be less than %d %s";
+    case TSDB_CODE_PAR_INTER_VALUE_TOO_BIG:
+      return "Interval cannot be more than %d %s";
    case TSDB_CODE_PAR_DB_NOT_SPECIFIED:
      return "Database not specified";
    case TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME:
--- a/source/libs/parser/src/sql.c
+++ b/source/libs/parser/src/sql.c
--- a/source/libs/parser/test/parInitialCTest.cpp
+++ b/source/libs/parser/test/parInitialCTest.cpp
@ -247,7 +247,11 @@ TEST_F(ParserInitialCTest, createDatabase) {
      for (int32_t i = 0; i < expect.numOfRetensions; ++i) {
        SRetention* pReten = (SRetention*)taosArrayGet(req.pRetensions, i);
        SRetention* pExpectReten = (SRetention*)taosArrayGet(expect.pRetensions, i);
+        if(i == 0) {
+          ASSERT_EQ(pReten->freq, 0);
+        } else {
          ASSERT_EQ(pReten->freq, pExpectReten->freq);
+        }
        ASSERT_EQ(pReten->keep, pExpectReten->keep);
        ASSERT_EQ(pReten->freqUnit, pExpectReten->freqUnit);
        ASSERT_EQ(pReten->keepUnit, pExpectReten->keepUnit);
@ -304,7 +308,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
      "PAGESIZE 8 "
      "PRECISION 'ns' "
      "REPLICA 3 "
-      "RETENTIONS 15s:7d,1m:21d,15m:500d "
+      "RETENTIONS -:7d,1m:21d,15m:500d "
      //      "STRICT 'on' "
      "WAL_LEVEL 2 "
      "VGROUPS 100 "
@ -340,12 +344,12 @@ TEST_F(ParserInitialCTest, createDatabase) {
 TEST_F(ParserInitialCTest, createDatabaseSemanticCheck) {
  useDb("root", "test");

-  run("create database db2 retentions 0s:1d", TSDB_CODE_PAR_INVALID_DB_OPTION);
-  run("create database db2 retentions 10s:0d", TSDB_CODE_PAR_INVALID_DB_OPTION);
-  run("create database db2 retentions 1w:1d", TSDB_CODE_PAR_INVALID_DB_OPTION);
-  run("create database db2 retentions 1w:1n", TSDB_CODE_PAR_INVALID_DB_OPTION);
-  run("create database db2 retentions 15s:7d,15m:21d,10m:500d", TSDB_CODE_PAR_INVALID_DB_OPTION);
-  run("create database db2 retentions 15s:7d,5m:21d,10m:10d", TSDB_CODE_PAR_INVALID_DB_OPTION);
+  run("create database db2 retentions -:1d,0s:1d", TSDB_CODE_PAR_INVALID_DB_OPTION);
+  run("create database db2 retentions -:0d", TSDB_CODE_PAR_INVALID_DB_OPTION);
+  run("create database db2 retentions -:1d,1w:1d", TSDB_CODE_PAR_INVALID_DB_OPTION);
+  run("create database db2 retentions -:1n,1w:1d", TSDB_CODE_PAR_INVALID_DB_OPTION);
+  run("create database db2 retentions -:7d,15m:21d,10m:500d", TSDB_CODE_PAR_INVALID_DB_OPTION);
+  run("create database db2 retentions -:7d,5m:21d,10m:10d", TSDB_CODE_PAR_INVALID_DB_OPTION);
 }

 /*
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@ -1197,6 +1197,83 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
  return TSDB_CODE_SUCCESS;
 }

+#define TS_FORMAT_MAX_LEN 4096
+int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput) {
+  int64_t ts;
+  char *  tsStr = taosMemoryMalloc(TS_FORMAT_MAX_LEN);
+  char *  format = taosMemoryMalloc(TS_FORMAT_MAX_LEN);
+  int32_t len, code = TSDB_CODE_SUCCESS;
+  SArray *formats = NULL;
+
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) {
+      colDataSetNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    char *tsData = colDataGetData(pInput[0].columnData, i);
+    char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0);
+    len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(tsData));
+    strncpy(tsStr, varDataVal(tsData), len);
+    tsStr[len] = '\0';
+    len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData));
+    if (pInput[1].numOfRows > 1 || i == 0) {
+      strncpy(format, varDataVal(formatData), len);
+      format[len] = '\0';
+      if (formats) {
+        taosArrayDestroy(formats);
+        formats = NULL;
+      }
+    }
+    int32_t precision = pOutput->columnData->info.precision;
+    char    errMsg[128] = {0};
+    code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128);
+    if (code) {
+      qError("func to_timestamp failed %s", errMsg);
+      code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED;
+      break;
+    }
+    colDataSetVal(pOutput->columnData, i, (char *)&ts, false);
+  }
+  if (formats) taosArrayDestroy(formats);
+  taosMemoryFree(tsStr);
+  taosMemoryFree(format);
+  return code;
+}
+
+int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput) {
+  char *  format = taosMemoryMalloc(TS_FORMAT_MAX_LEN);
+  char *  out = taosMemoryCalloc(1, TS_FORMAT_MAX_LEN + VARSTR_HEADER_SIZE);
+  int32_t len;
+  SArray *formats = NULL;
+  for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
+    if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) {
+      colDataSetNULL(pOutput->columnData, i);
+      continue;
+    }
+
+    char *ts = colDataGetData(pInput[0].columnData, i);
+    char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0);
+    len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData));
+    if (pInput[1].numOfRows > 1 || i == 0) {
+      strncpy(format, varDataVal(formatData), len);
+      format[len] = '\0';
+      if (formats) {
+        taosArrayDestroy(formats);
+        formats = NULL;
+      }
+    }
+    int32_t precision = pInput[0].columnData->info.precision;
+    taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN);
+    varDataSetLen(out, strlen(varDataVal(out)));
+    colDataSetVal(pOutput->columnData, i, out, false);
+  }
+  if (formats) taosArrayDestroy(formats);
+  taosMemoryFree(format);
+  taosMemoryFree(out);
+  return TSDB_CODE_SUCCESS;
+}
+
 /** Time functions **/
 static int64_t offsetFromTz(char *timezone, int64_t factor) {
  char *minStr = &timezone[3];
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@ -52,6 +52,11 @@ extern "C" {
 #define stTrace(...) do { if (stDebugFlag & DEBUG_TRACE) { taosPrintLog("STM ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on

+typedef struct {
+  int8_t       type;
+  SSDataBlock* pBlock;
+} SStreamTrigger;
+
 typedef struct SStreamGlobalEnv {
  int8_t inited;
  void*  timer;
@ -122,7 +127,7 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask);
 int32_t streamTaskFillHistoryFinished(SStreamTask* pTask);
 int32_t streamTransferStateToStreamTask(SStreamTask* pTask);

-int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, int32_t quotaRate);
+int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate);
 STaskId streamTaskExtractKey(const SStreamTask* pTask);
 void    streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo);
 void    streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo);
--- a/source/libs/stream/inc/streamsm.h
+++ b/source/libs/stream/inc/streamsm.h
@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_STREAMSM_H
+#define TDENGINE_STREAMSM_H
+
+#include "tstream.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// moore finite state machine for stream task
+typedef struct SStreamTaskState {
+  ETaskStatus state;
+  char*       name;
+} SStreamTaskState;
+
+typedef int32_t (*__state_trans_fn)(SStreamTask*);
+typedef int32_t (*__state_trans_succ_fn)(SStreamTask*);
+
+typedef struct SAttachedEventInfo {
+  ETaskStatus      status;    // required status that this event can be handled
+  EStreamTaskEvent event;     // the delayed handled event
+} SAttachedEventInfo;
+
+typedef struct STaskStateTrans {
+  bool                  autoInvokeEndFn;
+  SStreamTaskState      state;
+  EStreamTaskEvent      event;
+  SStreamTaskState      next;
+  __state_trans_fn      pAction;
+  __state_trans_succ_fn pSuccAction;
+  SAttachedEventInfo    attachEvent;
+} STaskStateTrans;
+
+struct SStreamTaskSM {
+  SStreamTask*     pTask;
+  STaskStateTrans* pActiveTrans;
+  int64_t          startTs;
+  SStreamTaskState current;
+  struct {
+    SStreamTaskState state;
+    EStreamTaskEvent evt;
+  } prev;
+  // register the next handled event, if current state is not allowed to handle this event
+  SArray* pWaitingEventList;
+};
+
+typedef struct SStreamEventInfo {
+  EStreamTaskEvent event;
+  const char*      name;
+} SStreamEventInfo;
+
+SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
+void* streamDestroyStateMachine(SStreamTaskSM* pSM);
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_STREAMSM_H
--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@ -62,14 +62,14 @@ static void streamSchedByTimer(void* param, void* tmrId) {
  int32_t      nextTrigger = (int32_t)pTask->info.triggerParam;

  int8_t status = atomic_load_8(&pTask->schedInfo.status);
-  stDebug("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
+  stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);

-  if (streamTaskShouldStop(&pTask->status) || streamTaskShouldPause(&pTask->status)) {
+  if (streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) {
    stDebug("s-task:%s jump out of schedTimer", id);
    return;
  }

-  if (pTask->status.taskStatus == TASK_STATUS__CK) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) {
    stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger);
  } else {
    if (status == TASK_TRIGGER_STATUS__ACTIVE) {
@ -127,7 +127,7 @@ int32_t streamSchedExec(SStreamTask* pTask) {
    SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
    if (pRunReq == NULL) {
      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      /*int8_t status = */streamTaskSetSchedStatusInActive(pTask);
+      /*int8_t status = */streamTaskSetSchedStatusInactive(pTask);
      stError("failed to create msg to aunch s-task:%s, reason out of memory", pTask->id.idStr);
      return -1;
    }
@ -267,8 +267,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
  }

  // disable the data from upstream tasks
-  int8_t st = pTask->status.taskStatus;
-  if (st == TASK_STATUS__HALT) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__HALT) {
    status = TASK_INPUT_STATUS__BLOCKED;
  }

@ -297,7 +296,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S
  return 0;
 }

-void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputInfo.status, TASK_INPUT_STATUS__FAILED); }
+void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); }

 void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) {
  int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@ -1467,16 +1467,24 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_c

 void destroyRocksdbCfInst(RocksdbCfInst* inst) {
  int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]);
+  if (inst->pHandle) {
    for (int i = 0; i < cfLen; i++) {
      if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]);
    }
+    taosMemoryFree(inst->pHandle);
+  }

-  rocksdb_writeoptions_destroy(inst->wOpt);
-  inst->wOpt = NULL;
-
-  rocksdb_readoptions_destroy(inst->rOpt);
-  taosMemoryFree(inst->cfOpt);
+  if (inst->cfOpt) {
+    for (int i = 0; i < cfLen; i++) {
+      rocksdb_options_destroy(inst->cfOpt[i]);
+      rocksdb_block_based_options_destroy(((RocksdbCfParam*)inst->param)[i].tableOpt);
+    }
+    taosMemoryFreeClear(inst->cfOpt);
    taosMemoryFreeClear(inst->param);
+  }
+  if (inst->wOpt) rocksdb_writeoptions_destroy(inst->wOpt);
+  if (inst->rOpt) rocksdb_readoptions_destroy(inst->rOpt);
+
  taosMemoryFree(inst);
 }

@ -1645,6 +1653,13 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
    pState->pTdbState->backendCfWrapperId = id;
    pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper;
    stInfo("succ to open state %p on backendWrapper, %p, %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr);
+
+    inst->pHandle = NULL;
+    inst->cfOpt = NULL;
+    inst->param = NULL;
+
+    inst->wOpt = NULL;
+    inst->rOpt = NULL;
    return 0;
  }
  taosThreadMutexUnlock(&handle->cfMutex);
@ -1992,14 +2007,12 @@ int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* ke
  return 0;
 }
 int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur) {
-  stDebug("streamStateCurPrev_rocksdb");
  if (!pCur) return -1;

  rocksdb_iter_prev(pCur->iter);
  return 0;
 }
 int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
-  stDebug("streamStateGetKVByCur_rocksdb");
  if (!pCur) return -1;
  SStateKey  tkey;
  SStateKey* pKtmp = &tkey;
@ -2024,7 +2037,6 @@ int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, cons
  return -1;
 }
 SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key) {
-  stDebug("streamStateGetAndCheckCur_rocksdb");
  SStreamStateCur* pCur = streamStateFillGetCur_rocksdb(pState, key);
  if (pCur) {
    int32_t code = streamStateGetGroupKVByCur_rocksdb(pCur, key, NULL, 0);
@ -2035,7 +2047,6 @@ SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey
 }

 SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) {
-  stDebug("streamStateSeekKeyNext_rocksdb");
  SStreamStateCur* pCur = createStreamStateCursor();
  if (pCur == NULL) {
    return NULL;
@ -2074,7 +2085,6 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin
 }

 SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
-  stDebug("streamStateGetCur_rocksdb");
  int32_t code = 0;

  const SStateKey maxStateKey = {.key = {.groupId = UINT64_MAX, .ts = INT64_MAX}, .opNum = INT64_MAX};
@ -2115,7 +2125,6 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
 }

 SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) {
-  stDebug("streamStateGetCur_rocksdb");
  SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper;

  SStreamStateCur* pCur = createStreamStateCursor();
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@ -135,12 +135,13 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) {
  ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);

+  // todo this status may not be set here.
  // 1. set task status to be prepared for check point, no data are allowed to put into inputQ.
-  pTask->status.taskStatus = TASK_STATUS__CK;
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
+
  pTask->checkpointingId = pReq->checkpointId;
  pTask->checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask);
  pTask->chkInfo.startTs = taosGetTimestampMs();
-
  pTask->execInfo.checkpoint += 1;

  // 2. Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task
@ -171,11 +172,15 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
  const char* id = pTask->id.idStr;
  int32_t code = TSDB_CODE_SUCCESS;

-  // set the task status
-  pTask->checkpointingId = checkpointId;
-
  // set task status
-  pTask->status.taskStatus = TASK_STATUS__CK;
+  if (streamTaskGetStatus(pTask, NULL) != TASK_STATUS__CK) {
+    pTask->checkpointingId = checkpointId;
+    code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
+    if (code != TSDB_CODE_SUCCESS) {
+      stError("s-task:%s handle checkpoint-trigger block failed, code:%s", id, tstrerror(code));
+      return code;
+    }
+  }

  { // todo: remove this when the pipeline checkpoint generating is used.
    SStreamMeta* pMeta = pTask->pMeta;
@ -195,6 +200,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
      stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
      continueDispatchCheckpointBlock(pBlock, pTask);
    } else {  // only one task exists, no need to dispatch downstream info
+      atomic_add_fetch_32(&pTask->checkpointNotReadyTasks, 1);
      streamProcessCheckpointReadyMsg(pTask);
      streamFreeQitem((SStreamQueueItem*)pBlock);
    }
@ -274,6 +280,8 @@ void streamTaskClearCheckInfo(SStreamTask* pTask) {
 }

 int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) {
+  int32_t vgId = pMeta->vgId;
+
  taosWLockLatch(&pMeta->lock);

  for (int32_t i = 0; i < taosArrayGetSize(pMeta->pTaskList); ++i) {
@ -288,21 +296,28 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) {
      continue;
    }

-    int8_t prev = p->status.taskStatus;
    ASSERT(p->chkInfo.checkpointId < p->checkpointingId && p->checkpointingId == checkpointId);

    p->chkInfo.checkpointId = p->checkpointingId;
    streamTaskClearCheckInfo(p);
-    streamSetStatusNormal(p);

-    // save the task
+    char* str = NULL;
+    streamTaskGetStatus(p, &str);
+
+    int32_t code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
+    if (code != TSDB_CODE_SUCCESS) {
+      stDebug("s-task:%s vgId:%d save task status failed, since handle event failed", p->id.idStr, vgId);
+      taosWUnLockLatch(&pMeta->lock);
+      return -1;
+    } else { // save the task
      streamMetaSaveTask(pMeta, p);
+    }

    stDebug(
        "vgId:%d s-task:%s level:%d open upstream inputQ, commit task status after checkpoint completed, "
        "checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status to be normal, prev:%s",
        pMeta->vgId, p->id.idStr, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer,
-        streamGetTaskStatusStr(prev));
+        str);
  }

  if (streamMetaCommit(pMeta) < 0) {
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@ -420,7 +420,7 @@ static void doRetryDispatchData(void* param, void* tmrId) {
  const char*  id = pTask->id.idStr;
  int32_t      msgId = pTask->execInfo.dispatch;

-  if (streamTaskShouldStop(&pTask->status)) {
+  if (streamTaskShouldStop(pTask)) {
    int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
    stDebug("s-task:%s should stop, abort from timer, ref:%d", pTask->id.idStr, ref);
    return;
@ -474,10 +474,10 @@ static void doRetryDispatchData(void* param, void* tmrId) {
  }

  if (code != TSDB_CODE_SUCCESS) {
-    if (!streamTaskShouldStop(&pTask->status)) {
+    if (!streamTaskShouldStop(pTask)) {
 //      stDebug("s-task:%s reset the waitRspCnt to be 0 before launch retry dispatch", pTask->id.idStr);
 //      atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0);
-      if (streamTaskShouldPause(&pTask->status)) {
+      if (streamTaskShouldPause(pTask)) {
        streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS * 10);
      } else {
        streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS);
@ -662,8 +662,10 @@ int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) {
    int32_t numOfVgs = taosArrayGetSize(vgInfo);
    pTask->notReadyTasks = numOfVgs;

+    char* p = NULL;
+    streamTaskGetStatus(pTask, &p);
    stDebug("s-task:%s send scan-history data complete msg to downstream (shuffle-dispatch) %d tasks, status:%s", pTask->id.idStr,
-           numOfVgs, streamGetTaskStatusStr(pTask->status.taskStatus));
+           numOfVgs, p);
    for (int32_t i = 0; i < numOfVgs; i++) {
      SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
      req.downstreamTaskId = pVgInfo->taskId;
@ -775,8 +777,9 @@ int32_t doDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHist
  initRpcMsg(&msg, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, buf, tlen + sizeof(SMsgHead));

  tmsgSendReq(pEpSet, &msg);
-  const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
-  stDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, pStatus,
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+  stDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, p,
         pReq->downstreamTaskId, vgId);
  return 0;
 }
@ -1008,7 +1011,7 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo,
  taosThreadMutexUnlock(&pTask->lock);

  int32_t num = taosArrayGetSize(pTask->pRspMsgList);
-  stDebug("s-task:%s add scan history finish rsp msg for task:0x%x, total:%d", pTask->id.idStr, pReq->upstreamTaskId,
+  stDebug("s-task:%s add scan-history finish rsp msg for task:0x%x, total:%d", pTask->id.idStr, pReq->upstreamTaskId,
         num);
  return TSDB_CODE_SUCCESS;
 }
@ -1024,7 +1027,7 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask) {
    SStreamContinueExecInfo* pInfo = taosArrayGet(pTask->pRspMsgList, i);
    tmsgSendRsp(&pInfo->msg);

-    stDebug("s-task:%s level:%d notify upstream:0x%x continuing scan data in WAL", id, level, pInfo->taskId);
+    stDebug("s-task:%s level:%d notify upstream:0x%x continuing handle data in WAL", id, level, pInfo->taskId);
  }

  taosArrayClear(pTask->pRspMsgList);
@ -1040,8 +1043,8 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId
  int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs;

  // put data into inputQ of current task is also allowed
-  if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
-    pTask->inputInfo.status = TASK_INPUT_STATUS__NORMAL;
+  if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
+    pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
    stDebug("s-task:%s downstream task:0x%x resume to normal from inputQ blocking, blocking time:%" PRId64 "ms",
            pTask->id.idStr, downstreamId, el);
  } else {
@ -1093,7 +1096,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i

  } else {  // code == 0
    if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) {
-      pTask->inputInfo.status = TASK_INPUT_STATUS__BLOCKED;
+      pTask->inputq.status = TASK_INPUT_STATUS__BLOCKED;
      // block the input of current task, to push pressure to upstream
      taosThreadMutexLock(&pTask->lock);
      taosArrayPush(pTask->msgInfo.pRetryList, &pRsp->downstreamNodeId);
@ -1113,16 +1116,16 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
    ASSERT(leftRsp >= 0);

    if (leftRsp > 0) {
-      stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%d, waiting for %d rsp",
-               id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, code, leftRsp);
+      stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, waiting for %d rsp",
+               id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), leftRsp);
    } else {
      stDebug(
-          "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%d, all rsp",
-          id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, code);
+          "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all rsp",
+          id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code));
    }
  } else {
-    stDebug("s-task:%s recv fix-dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%d",
-            id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, code);
+    stDebug("s-task:%s recv fix-dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s",
+            id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code));
  }

  ASSERT(leftRsp >= 0);
@ -1181,6 +1184,9 @@ int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpda
    if (tEncodeSEpSet(pEncoder, &pInfo->prevEp) < 0) return -1;
    if (tEncodeSEpSet(pEncoder, &pInfo->newEp) < 0) return -1;
  }
+
+  // todo this new attribute will be result in being incompatible with previous version
+  if (tEncodeI32(pEncoder, pMsg->transId) < 0) return -1;
  tEndEncode(pEncoder);
  return pEncoder->pos;
 }
@ -1201,6 +1207,8 @@ int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg*
    taosArrayPush(pMsg->pNodeList, &info);
  }

+  if (tDecodeI32(pDecoder, &pMsg->transId) < 0) return -1;
+
  tEndDecode(pDecoder);
  return 0;
 }
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@ -22,14 +22,13 @@

 static int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask);

-bool streamTaskShouldStop(const SStreamStatus* pStatus) {
-  int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus);
-  return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING);
+bool streamTaskShouldStop(const SStreamTask* pTask) {
+  ETaskStatus s = streamTaskGetStatus(pTask, NULL);
+  return (s == TASK_STATUS__STOP) || (s == TASK_STATUS__DROPPING);
 }

-bool streamTaskShouldPause(const SStreamStatus* pStatus) {
-  int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus);
-  return (status == TASK_STATUS__PAUSE);
+bool streamTaskShouldPause(const SStreamTask* pTask) {
+  return (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE);
 }

 static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBlock) {
@ -102,12 +101,12 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i
      pRes = taosArrayInit(4, sizeof(SSDataBlock));
    }

-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask)) {
      taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
      return 0;
    }

-    if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+    if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
      stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry exec task", pTask->id.idStr);
      taosMsleep(1000);
      continue;
@ -198,7 +197,7 @@ int32_t streamScanHistoryData(SStreamTask* pTask) {
  qSetStreamOpOpen(exec);

  while (!finished) {
-    if (streamTaskShouldPause(&pTask->status)) {
+    if (streamTaskShouldPause(pTask)) {
      double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
      stDebug("s-task:%s paused from the scan-history task, elapsed time:%.2fsec", pTask->id.idStr, el);
      break;
@ -213,12 +212,12 @@ int32_t streamScanHistoryData(SStreamTask* pTask) {
    int32_t size = 0;
    int32_t numOfBlocks = 0;
    while (1) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
        taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
        return 0;
      }

-      if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+      if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
        stDebug("s-task:%s inputQ is blocked, wait for 10sec and retry", pTask->id.idStr);
        taosMsleep(10000);
        continue;
@ -309,21 +308,19 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
           pStreamTask->id.idStr);
  }

-  ASSERT(((pStreamTask->status.taskStatus == TASK_STATUS__STOP) ||
-          (pStreamTask->hTaskInfo.id.taskId == pTask->id.taskId)) &&
+  ETaskStatus status = streamTaskGetStatus(pStreamTask, NULL);
+  ASSERT(((status == TASK_STATUS__DROPPING) || (pStreamTask->hTaskInfo.id.taskId == pTask->id.taskId)) &&
         pTask->status.appendTranstateBlock == true);

  STimeWindow* pTimeWindow = &pStreamTask->dataRange.window;

-  // todo. the dropping status should be append to the status after the halt completed.
  // It must be halted for a source stream task, since when the related scan-history-data task start scan the history
  // for the step 2.
-  int8_t status = pStreamTask->status.taskStatus;
  if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING);
+    ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP);
  } else {
-    ASSERT(status == TASK_STATUS__NORMAL);
-    pStreamTask->status.taskStatus = TASK_STATUS__HALT;
+    ASSERT(status == TASK_STATUS__READY|| status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP);
+    streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
    stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr);
  }

@ -333,13 +330,14 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
  // In case of sink tasks, no need to halt them.
  // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to
  // start the task state transfer procedure.
-  // When a task is idle with halt status, all data in inputQ are consumed.
+  char* p = NULL;
+  streamTaskGetStatus(pStreamTask, &p);
  if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) {
    // update the scan data range for source task.
    stDebug("s-task:%s level:%d stream task window %" PRId64 " - %" PRId64 " update to %" PRId64 " - %" PRId64
            ", status:%s, sched-status:%d",
            pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN,
-           pTimeWindow->ekey, streamGetTaskStatusStr(TASK_STATUS__NORMAL), pStreamTask->status.schedStatus);
+            pTimeWindow->ekey, p, pStreamTask->status.schedStatus);
  } else {
    stDebug("s-task:%s no need to update time window for non-source task", pStreamTask->id.idStr);
  }
@ -354,7 +352,7 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {

  // 3. resume the state of stream task, after this function, the stream task will run immidately. But it can not be
  // pause, since the pause allowed attribute is not set yet.
-  streamTaskResumeFromHalt(pStreamTask);
+  streamTaskResume(pStreamTask);  // todo refactor: use streamTaskResume.

  stDebug("s-task:%s fill-history task set status to be dropping, save the state into disk", pTask->id.idStr);

@ -362,20 +360,21 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
  streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id);

  // 5. clear the link between fill-history task and stream task info
-  pStreamTask->hTaskInfo.id.taskId = 0;
-  pStreamTask->hTaskInfo.id.streamId = 0;
+//  CLEAR_RELATED_FILLHISTORY_TASK(pStreamTask);

  // 6. save to disk
  taosWLockLatch(&pMeta->lock);
-  streamMetaSaveTask(pMeta, pStreamTask);
-  if (streamMetaCommit(pMeta) < 0) {
+
+  pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask, NULL);
+//  streamMetaSaveTask(pMeta, pStreamTask);
+//  if (streamMetaCommit(pMeta) < 0) {
    // persist to disk
-  }
+//  }
  taosWUnLockLatch(&pMeta->lock);

  // 7. pause allowed.
  streamTaskEnablePause(pStreamTask);
-  if (taosQueueEmpty(pStreamTask->inputInfo.queue->pQueue)) {
+  if ((pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) && taosQueueEmpty(pStreamTask->inputq.queue->pQueue)) {
    SStreamRefDataBlock* pItem = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);

    SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA);
@ -492,7 +491,7 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock
      code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
      if (code == 0) {
        streamDispatchStreamBlock(pTask);
-      } else {
+      } else {  // todo put into queue failed, retry
        streamFreeQitem((SStreamQueueItem*)pBlock);
      }
    } else {  // level == TASK_LEVEL__SINK
@ -500,12 +499,12 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock
    }
  } else {  // non-dispatch task, do task state transfer directly
    streamFreeQitem((SStreamQueueItem*)pBlock);
-    stDebug("s-task:%s non-dispatch task, start to transfer state directly", id);
+    stDebug("s-task:%s non-dispatch task, level:%d start to transfer state directly", id, pTask->info.taskLevel);
    ASSERT(pTask->info.fillHistory == 1);
-    code = streamTransferStateToStreamTask(pTask);

+    code = streamTransferStateToStreamTask(pTask);
    if (code != TSDB_CODE_SUCCESS) {
-      /*int8_t status = */ streamTaskSetSchedStatusInActive(pTask);
+      /*int8_t status = */ streamTaskSetSchedStatusInactive(pTask);
    }
  }

@ -526,7 +525,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
    int32_t           blockSize = 0;
    int32_t           numOfBlocks = 0;
    SStreamQueueItem* pInput = NULL;
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__UNINIT)) {
      stDebug("s-task:%s stream task is stopped", id);
      break;
    }
@ -592,8 +591,9 @@ int32_t streamExecForAll(SStreamTask* pTask) {
    // todo other thread may change the status
    // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
    if (type == STREAM_INPUT__CHECKPOINT) {
-      stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr,
-              streamGetTaskStatusStr(pTask->status.taskStatus));
+      char* p = NULL;
+      streamTaskGetStatus(pTask, &p);
+      stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p);
      streamTaskBuildCheckpoint(pTask);
      return 0;
    }
@ -605,8 +605,15 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 // the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not
 // be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING.
 bool streamTaskIsIdle(const SStreamTask* pTask) {
-  return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || pTask->status.taskStatus == TASK_STATUS__STOP ||
-          pTask->status.taskStatus == TASK_STATUS__DROPPING);
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
+  return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || status == TASK_STATUS__STOP ||
+          status == TASK_STATUS__DROPPING);
+}
+
+bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) {
+  ETaskStatus st = streamTaskGetStatus(pTask, NULL);
+  return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__STREAM_SCAN_HISTORY ||
+          st == TASK_STATUS__CK);
 }

 int32_t streamExecTask(SStreamTask* pTask) {
@ -623,20 +630,23 @@ int32_t streamExecTask(SStreamTask* pTask) {
      }

      taosThreadMutexLock(&pTask->lock);
-      if ((streamQueueGetNumOfItems(pTask->inputInfo.queue) == 0) || streamTaskShouldStop(&pTask->status) ||
-          streamTaskShouldPause(&pTask->status)) {
+      if ((streamQueueGetNumOfItems(pTask->inputq.queue) == 0) || streamTaskShouldStop(pTask) ||
+          streamTaskShouldPause(pTask)) {
        atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
        taosThreadMutexUnlock(&pTask->lock);

-        stDebug("s-task:%s exec completed, status:%s, sched-status:%d", id,
-               streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus);
+        char* p = NULL;
+        streamTaskGetStatus(pTask, &p);
+        stDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, p, pTask->status.schedStatus);
        return 0;
      }
      taosThreadMutexUnlock(&pTask->lock);
    }
  } else {
-    stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id,
-           streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus);
+    char* p = NULL;
+    streamTaskGetStatus(pTask, &p);
+    stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p,
+            pTask->status.schedStatus);
  }

  return 0;
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@ -143,8 +143,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
    goto _err;
  }

-  pMeta->pUpdateTaskSet = taosHashInit(64, fp, false, HASH_NO_LOCK);
-  if (pMeta->pUpdateTaskSet == NULL) {
+  pMeta->updateInfo.pTasks = taosHashInit(64, fp, false, HASH_NO_LOCK);
+  if (pMeta->updateInfo.pTasks == NULL) {
    goto _err;
  }

@ -211,7 +211,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
         stage);
  return pMeta;

-_err:
+  _err:
  taosMemoryFree(pMeta->path);
  if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap);
  if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList);
@ -219,7 +219,7 @@ _err:
  if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb);
  if (pMeta->db) tdbClose(pMeta->db);
  if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo);
-  if (pMeta->pUpdateTaskSet) taosHashCleanup(pMeta->pUpdateTaskSet);
+  if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks);
  if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
  taosMemoryFree(pMeta);

@ -228,11 +228,15 @@ _err:
 }

 int32_t streamMetaReopen(SStreamMeta* pMeta) {
+  // backup the restart flag
+  int32_t restartFlag = pMeta->startInfo.startAllTasksFlag;
  streamMetaClear(pMeta);

+  pMeta->startInfo.startAllTasksFlag = restartFlag;
+
+  // NOTE: role should not be changed during reopen meta
  pMeta->streamBackendRid = -1;
  pMeta->streamBackend = NULL;
-  pMeta->role = NODE_ROLE_UNINIT;

  char* defaultPath = taosMemoryCalloc(1, strlen(pMeta->path) + 128);
  sprintf(defaultPath, "%s%s%s", pMeta->path, TD_DIRSEP, "state");
@ -274,6 +278,7 @@ int32_t streamMetaReopen(SStreamMeta* pMeta) {
 }

 void streamMetaClear(SStreamMeta* pMeta) {
+  // remove all existed tasks in this vnode
  void* pIter = NULL;
  while ((pIter = taosHashIterate(pMeta->pTasksMap, pIter)) != NULL) {
    SStreamTask* p = *(SStreamTask**)pIter;
@ -299,6 +304,9 @@ void streamMetaClear(SStreamMeta* pMeta) {
  taosArrayClear(pMeta->chkpInUse);
  pMeta->numOfStreamTasks = 0;
  pMeta->numOfPausedTasks = 0;
+  pMeta->chkptNotReadyTasks = 0;
+
+  streamMetaResetStartInfo(&pMeta->startInfo);
 }

 void streamMetaClose(SStreamMeta* pMeta) {
@ -336,7 +344,7 @@ void streamMetaCloseImpl(void* arg) {

  taosHashCleanup(pMeta->pTasksMap);
  taosHashCleanup(pMeta->pTaskBackendUnique);
-  taosHashCleanup(pMeta->pUpdateTaskSet);
+  taosHashCleanup(pMeta->updateInfo.pTasks);
  taosHashCleanup(pMeta->startInfo.pReadyTaskSet);

  taosMemoryFree(pMeta->pHbInfo);
@ -348,6 +356,7 @@ void streamMetaCloseImpl(void* arg) {
  stDebug("end to close stream meta");
 }

+// todo let's check the status for each task
 int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
  void*   buf = NULL;
  int32_t len;
@ -437,31 +446,13 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) {
  return (int32_t)size;
 }

-int32_t streamMetaGetNumOfStreamTasks(SStreamMeta* pMeta) {
-  int32_t num = 0;
-  size_t  size = taosArrayGetSize(pMeta->pTaskList);
-  for (int32_t i = 0; i < size; ++i) {
-    STaskId*      pId = taosArrayGet(pMeta->pTaskList, i);
-    SStreamTask** p = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId));
-    if (p == NULL) {
-      continue;
-    }
-
-    if ((*p)->info.fillHistory == 0) {
-      num += 1;
-    }
-  }
-
-  return num;
-}
-
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) {
  taosRLockLatch(&pMeta->lock);

  STaskId       id = {.streamId = streamId, .taskId = taskId};
  SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
  if (ppTask != NULL) {
-    if (!streamTaskShouldStop(&(*ppTask)->status)) {
+    if (!streamTaskShouldStop(*ppTask)) {
      int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
      taosRUnLockLatch(&pMeta->lock);
      stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref);
@ -478,7 +469,7 @@ void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask)
  if (ref > 0) {
    stTrace("s-task:%s release task, ref:%d", pTask->id.idStr, ref);
  } else if (ref == 0) {
-    ASSERT(streamTaskShouldStop(&pTask->status));
+    ASSERT(streamTaskShouldStop(pTask));
    stTrace("s-task:%s all refs are gone, free it", pTask->id.idStr);
    tFreeStreamTask(pTask);
  } else if (ref < 0) {
@ -506,11 +497,15 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
  SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
  if (ppTask) {
    pTask = *ppTask;
-    if (streamTaskShouldPause(&pTask->status)) {
+
+    // desc the paused task counter
+    if (streamTaskShouldPause(pTask)) {
      int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
      stInfo("vgId:%d s-task:%s drop stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
    }
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
+
+    // handle the dropping event
+    streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_DROPPING);
  } else {
    stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId);
    taosWUnLockLatch(&pMeta->lock);
@ -518,13 +513,12 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
  }
  taosWUnLockLatch(&pMeta->lock);

-  stDebug("s-task:0x%x set task status:%s and start to unregister it", taskId,
-          streamGetTaskStatusStr(TASK_STATUS__DROPPING));
+  stDebug("s-task:0x%x set task status:dropping and start to unregister it", taskId);

  while (1) {
    taosRLockLatch(&pMeta->lock);
-    ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));

+    ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
    if (ppTask) {
      if ((*ppTask)->status.timerActive == 0) {
        taosRUnLockLatch(&pMeta->lock);
@ -544,23 +538,19 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
  taosWLockLatch(&pMeta->lock);
  ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
  if (ppTask) {
+    pTask = *ppTask;
+
    // it is an fill-history task, remove the related stream task's id that points to it
-    if ((*ppTask)->info.fillHistory == 1) {
-      STaskId streamTaskId = {.streamId = (*ppTask)->streamTaskId.streamId, .taskId = (*ppTask)->streamTaskId.taskId};
-      SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &streamTaskId, sizeof(streamTaskId));
-      if (ppStreamTask != NULL) {
-        (*ppStreamTask)->hTaskInfo.id.taskId = 0;
-        (*ppStreamTask)->hTaskInfo.id.streamId = 0;
-      }
+    if (pTask->info.fillHistory == 1) {
+      streamTaskClearHTaskAttr(pTask);
    } else {
      atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1);
    }

    taosHashRemove(pMeta->pTasksMap, &id, sizeof(id));
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
+    doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id);

    ASSERT(pTask->status.timerActive == 0);
-    doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id);

    if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) {
      stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt);
@ -691,7 +681,8 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) {
      tFreeStreamTask(pTask);
      stError(
          "vgId:%d stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild "
-          "stream manually", vgId, tsDataDir);
+          "stream manually",
+          vgId, tsDataDir);
      return -1;
    }
    tDecoderClear(&decoder);
@ -701,8 +692,8 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) {
      tFreeStreamTask(pTask);

      STaskId id = streamTaskExtractKey(pTask);
-
      taosArrayPush(pRecycleList, &id);
+
      int32_t total = taosArrayGetSize(pRecycleList);
      stDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total);
      continue;
@ -738,7 +729,7 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta) {
      atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1);
    }

-    if (streamTaskShouldPause(&pTask->status)) {
+    if (streamTaskShouldPause(pTask)) {
      atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
    }

@ -790,6 +781,15 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
    if (tEncodeI64(pEncoder, ps->activeCheckpointId) < 0) return -1;
    if (tEncodeI8(pEncoder, ps->checkpointFailed) < 0) return -1;
  }
+
+  int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes);
+  if (tEncodeI32(pEncoder, numOfVgs) < 0) return -1;
+
+  for (int j = 0; j < numOfVgs; ++j) {
+    int32_t* pVgId = taosArrayGet(pReq->pUpdateNodes, j);
+    if (tEncodeI32(pEncoder, *pVgId) < 0) return -1;
+  }
+
  tEndEncode(pEncoder);
  return pEncoder->pos;
 }
@ -823,6 +823,17 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
    taosArrayPush(pReq->pTaskStatus, &entry);
  }

+  int32_t numOfVgs = 0;
+  if (tDecodeI32(pDecoder, &numOfVgs) < 0) return -1;
+
+  pReq->pUpdateNodes = taosArrayInit(numOfVgs, sizeof(int32_t));
+
+  for (int j = 0; j < numOfVgs; ++j) {
+    int32_t vgId = 0;
+    if (tDecodeI32(pDecoder, &vgId) < 0) return -1;
+    taosArrayPush(pReq->pUpdateNodes, &vgId);
+  }
+
  tEndDecode(pDecoder);
  return 0;
 }
@ -852,8 +863,8 @@ void metaHbToMnode(void* param, void* tmrId) {
  }

  // not leader not send msg
-  if (pMeta->role == NODE_ROLE_FOLLOWER) {
-    stInfo("vgId:%d follower not send hb to mnode", pMeta->vgId);
+  if (pMeta->role != NODE_ROLE_LEADER) {
+    stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role);
    taosReleaseRef(streamMetaId, rid);
    pMeta->pHbInfo->hbStart = 0;
    return;
@ -877,13 +888,14 @@ void metaHbToMnode(void* param, void* tmrId) {
  int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);

  SEpSet epset = {0};
-  bool   hasValEpset = false;
+  bool   hasMnodeEpset = false;
+
  hbMsg.vgId = pMeta->vgId;
  hbMsg.pTaskStatus = taosArrayInit(numOfTasks, sizeof(STaskStatusEntry));
+  hbMsg.pUpdateNodes = taosArrayInit(numOfTasks, sizeof(int32_t));

  for (int32_t i = 0; i < numOfTasks; ++i) {
    STaskId*      pId = taosArrayGet(pMeta->pTaskList, i);
-
    SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId));

    // not report the status of fill-history task
@ -893,10 +905,10 @@ void metaHbToMnode(void* param, void* tmrId) {

    STaskStatusEntry entry = {
        .id = *pId,
-        .status = (*pTask)->status.taskStatus,
+        .status = streamTaskGetStatus(*pTask, NULL),
        .nodeId = pMeta->vgId,
        .stage = pMeta->stage,
-        .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputInfo.queue)),
+        .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)),
    };

    entry.inputRate = entry.inputQUsed*100.0/STREAM_TASK_QUEUE_CAPACITY_IN_SIZE;
@ -915,18 +927,39 @@ void metaHbToMnode(void* param, void* tmrId) {
      walReaderValidVersionRange((*pTask)->exec.pWalReader, &entry.verStart, &entry.verEnd);
    }

-    taosArrayPush(hbMsg.pTaskStatus, &entry);
+    taosThreadMutexLock(&(*pTask)->lock);
+    int32_t num = taosArrayGetSize((*pTask)->outputInfo.pDownstreamUpdateList);
+    for (int j = 0; j < num; ++j) {
+      int32_t *pNodeId = taosArrayGet((*pTask)->outputInfo.pDownstreamUpdateList, j);

-    if (!hasValEpset) {
+      bool exist = false;
+      int32_t numOfExisted = taosArrayGetSize(hbMsg.pUpdateNodes);
+      for (int k = 0; k < numOfExisted; ++k) {
+        if (*pNodeId == *(int32_t*)taosArrayGet(hbMsg.pUpdateNodes, k)) {
+          exist = true;
+          break;
+        }
+      }
+
+      if (!exist) {
+        taosArrayPush(hbMsg.pUpdateNodes, pNodeId);
+      }
+    }
+
+    taosArrayClear((*pTask)->outputInfo.pDownstreamUpdateList);
+    taosThreadMutexUnlock(&(*pTask)->lock);
+
+    taosArrayPush(hbMsg.pTaskStatus, &entry);
+    if (!hasMnodeEpset) {
      epsetAssign(&epset, &(*pTask)->info.mnodeEpset);
-      hasValEpset = true;
+      hasMnodeEpset = true;
    }
  }

  hbMsg.numOfTasks = taosArrayGetSize(hbMsg.pTaskStatus);
  taosRUnLockLatch(&pMeta->lock);

-  if (hasValEpset) {
+  if (hasMnodeEpset) {
    int32_t code = 0;
    int32_t tlen = 0;

@ -971,13 +1004,14 @@ void metaHbToMnode(void* param, void* tmrId) {
  }

  taosArrayDestroy(hbMsg.pTaskStatus);
+  taosArrayDestroy(hbMsg.pUpdateNodes);
+
  taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamEnv.timer, &pMeta->pHbInfo->hbTmr);
  taosReleaseRef(streamMetaId, rid);
 }

-static bool hasStreamTaskInTimer(SStreamMeta* pMeta) {
+bool streamMetaTaskInTimer(SStreamMeta* pMeta) {
  bool inTimer = false;
-
  taosWLockLatch(&pMeta->lock);

  void* pIter = NULL;
@ -1031,7 +1065,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) {
  stDebug("vgId:%d start to check all tasks", vgId);
  int64_t st = taosGetTimestampMs();

-  while (hasStreamTaskInTimer(pMeta)) {
+  while (streamMetaTaskInTimer(pMeta)) {
    stDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
    taosMsleep(100);
  }
@ -1054,5 +1088,6 @@ void streamMetaInitForSnode(SStreamMeta* pMeta) {

 void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) {
  taosHashClear(pStartInfo->pReadyTaskSet);
-  pStartInfo->startedAfterNodeUpdate = 0;
+  pStartInfo->startAllTasksFlag = 0;
+  pStartInfo->readyTs = 0;
 }
--- a/source/libs/stream/src/streamQueue.c
+++ b/source/libs/stream/src/streamQueue.c
@ -18,7 +18,7 @@
 #define MAX_STREAM_EXEC_BATCH_NUM                 32
 #define MAX_SMOOTH_BURST_RATIO                    5     // 5 sec
 #define WAIT_FOR_DURATION                         40
-#define SINK_TASK_IDLE_DURATION                   200   // 200 ms
+#define OUTPUT_QUEUE_FULL_WAIT_DURATION           500   // 500 ms

 // todo refactor:
 // read data from input queue
@ -119,17 +119,8 @@ int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue) {
  return numOfItems1 + numOfItems2;
 }

-int32_t streamQueueGetAvailableSpace(const SStreamQueue* pQueue, int32_t* availNum, double* availSize) {
-  int32_t num = streamQueueGetNumOfItems(pQueue);
-  *availNum = STREAM_TASK_QUEUE_CAPACITY - num;
-
-  *availSize = STREAM_TASK_QUEUE_CAPACITY_IN_SIZE - taosQueueMemorySize(pQueue->pQueue);
-  return 0;
-}
-
-// todo: fix it: data in Qall is not included here
 int32_t streamQueueGetItemSize(const SStreamQueue* pQueue) {
-  return taosQueueMemorySize(pQueue->pQueue);
+  return taosQueueMemorySize(pQueue->pQueue) + taosQallUnAccessedMemSize(pQueue->qall);
 }

 int32_t streamQueueItemGetSize(const SStreamQueueItem* pItem) {
@ -173,12 +164,12 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
  }

  while (1) {
-    if (streamTaskShouldPause(&pTask->status) || streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldPause(pTask) || streamTaskShouldStop(pTask)) {
      stDebug("s-task:%s task should pause, extract input blocks:%d", pTask->id.idStr, *numOfBlocks);
      return TSDB_CODE_SUCCESS;
    }

-    SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputInfo.queue);
+    SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue);
    if (qItem == NULL) {
      if ((taskLevel == TASK_LEVEL__SOURCE  || taskLevel == TASK_LEVEL__SINK) && (++retryTimes) < MAX_RETRY_TIMES) {
        taosMsleep(WAIT_FOR_DURATION);
@ -220,7 +211,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
          streamTaskConsumeQuota(pTask->outputInfo.pTokenBucket, *blockSize);
        }

-        streamQueueProcessFail(pTask->inputInfo.queue);
+        streamQueueProcessFail(pTask->inputq.queue);
        return TSDB_CODE_SUCCESS;
      }
    } else {
@ -241,7 +232,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
            streamTaskConsumeQuota(pTask->outputInfo.pTokenBucket, *blockSize);
          }

-          streamQueueProcessFail(pTask->inputInfo.queue);
+          streamQueueProcessFail(pTask->inputq.queue);
          return TSDB_CODE_SUCCESS;
        }

@ -249,7 +240,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
      }

      *numOfBlocks += 1;
-      streamQueueProcessSuccess(pTask->inputInfo.queue);
+      streamQueueProcessSuccess(pTask->inputq.queue);

      if (*numOfBlocks >= MAX_STREAM_EXEC_BATCH_NUM) {
        stDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM);
@ -267,12 +258,12 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu

 int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) {
  int8_t      type = pItem->type;
-  STaosQueue* pQueue = pTask->inputInfo.queue->pQueue;
-  int32_t     total = streamQueueGetNumOfItems(pTask->inputInfo.queue) + 1;
+  STaosQueue* pQueue = pTask->inputq.queue->pQueue;
+  int32_t     total = streamQueueGetNumOfItems(pTask->inputq.queue) + 1;

  if (type == STREAM_INPUT__DATA_SUBMIT) {
    SStreamDataSubmit* px = (SStreamDataSubmit*)pItem;
-    if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pTask->inputInfo.queue)) {
+    if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && streamQueueIsFull(pTask->inputq.queue)) {
      double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));
      stTrace(
          "s-task:%s inputQ is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data",
@ -299,7 +290,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
           msgLen, ver, total, size + SIZE_IN_MiB(msgLen));
  } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE ||
             type == STREAM_INPUT__REF_DATA_BLOCK) {
-    if (streamQueueIsFull(pTask->inputInfo.queue)) {
+    if (streamQueueIsFull(pTask->inputq.queue)) {
      double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));

      stTrace("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort",
@ -354,7 +345,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
  STaosQueue* pQueue = pTask->outputq.queue->pQueue;

  while (streamQueueIsFull(pTask->outputq.queue)) {
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask)) {
      stInfo("s-task:%s discard result block due to task stop", pTask->id.idStr);
      return TSDB_CODE_STREAM_EXEC_CANCELLED;
    }
@ -362,9 +353,10 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
    int32_t total = streamQueueGetNumOfItems(pTask->outputq.queue);
    double  size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));
    // let's wait for there are enough space to hold this result pBlock
-    stDebug("s-task:%s outputQ is full, wait for 500ms and retry, outputQ items:%d, size:%.2fMiB", pTask->id.idStr,
-           total, size);
-    taosMsleep(500);
+    stDebug("s-task:%s outputQ is full, wait for %dms and retry, outputQ items:%d, size:%.2fMiB", pTask->id.idStr,
+            OUTPUT_QUEUE_FULL_WAIT_DURATION, total, size);
+
+    taosMsleep(OUTPUT_QUEUE_FULL_WAIT_DURATION);
  }

  int32_t code = taosWriteQitem(pQueue, pBlock);
@ -381,7 +373,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
  return TSDB_CODE_SUCCESS;
 }

-int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, int32_t quotaRate) {
+int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate) {
  if (numCap < 10 || numRate < 10 || pBucket == NULL) {
    stError("failed to init sink task bucket, cap:%d, rate:%d", numCap, numRate);
    return TSDB_CODE_INVALID_PARA;
--- a/source/libs/stream/src/streamRecover.c
+++ b/source/libs/stream/src/streamRecover.c
@ -17,6 +17,7 @@
 #include "trpc.h"
 #include "ttimer.h"
 #include "wal.h"
+#include "streamsm.h"

 typedef struct SLaunchHTaskInfo {
  SStreamMeta* pMeta;
@ -34,16 +35,18 @@ static void              streamTaskSetRangeStreamCalc(SStreamTask* pTask);
 static int32_t           initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated);
 static SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 static void              tryLaunchHistoryTask(void* param, void* tmrId);
+static void              doProcessDownstreamReadyRsp(SStreamTask* pTask);

-static void streamTaskSetReady(SStreamTask* pTask, int32_t numOfReqs) {
-  SStreamMeta* pMeta = pTask->pMeta;
-  int32_t      vgId = pMeta->vgId;
+int32_t streamTaskSetReady(SStreamTask* pTask) {
+  char*       p = NULL;
+  int32_t     numOfDowns = streamTaskGetNumOfDownstream(pTask);
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);

-  if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY && pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
+  if ((status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY) &&
+      pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
    pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->upstreamInfo.pList);
    stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s",
-           pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream,
-           streamGetTaskStatusStr(pTask->status.taskStatus));
+           pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, p);
  }

  ASSERT(pTask->status.downstreamReady == 0);
@ -52,34 +55,10 @@ static void streamTaskSetReady(SStreamTask* pTask, int32_t numOfReqs) {
  pTask->execInfo.start = taosGetTimestampMs();
  int64_t el = (pTask->execInfo.start - pTask->execInfo.init);
  stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s",
-          pTask->id.idStr, numOfReqs, el, streamGetTaskStatusStr(pTask->status.taskStatus));
+          pTask->id.idStr, numOfDowns, el, p);

-  taosWLockLatch(&pMeta->lock);
-
-  STaskId id = streamTaskExtractKey(pTask);
-  taosHashPut(pMeta->startInfo.pReadyTaskSet, &id, sizeof(id), NULL, 0);
-
-  int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta);
-
-  if (taosHashGetSize(pMeta->startInfo.pReadyTaskSet) == numOfTotal) {
-    STaskStartInfo* pStartInfo = &pMeta->startInfo;
-    pStartInfo->readyTs = pTask->execInfo.start;
-
-    if (pStartInfo->startTs != 0) {
-      pStartInfo->elapsedTime = pStartInfo->readyTs - pStartInfo->startTs;
-    } else {
-      pStartInfo->elapsedTime = 0;
-    }
-
-    streamMetaResetStartInfo(pStartInfo);
-
-    stDebug("vgId:%d all %d task(s) are started successfully, last ready task:%s level:%d, startTs:%" PRId64
-            ", readyTs:%" PRId64 " total elapsed time:%.2fs",
-            vgId, numOfTotal, pTask->id.idStr, pTask->info.taskLevel, pStartInfo->startTs, pStartInfo->readyTs,
-            pStartInfo->elapsedTime / 1000.0);
-  }
-
-  taosWUnLockLatch(&pMeta->lock);
+  streamMetaUpdateTaskReadyInfo(pTask);
+  return TSDB_CODE_SUCCESS;
 }

 int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) {
@ -114,33 +93,28 @@ static int32_t doStartScanHistoryTask(SStreamTask* pTask) {
 }

 int32_t streamTaskStartScanHistory(SStreamTask* pTask) {
-  ASSERT(pTask->status.downstreamReady == 1);
+  int32_t level = pTask->info.taskLevel;
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);

-  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+  ASSERT(pTask->status.downstreamReady == 1 &&
+         ((status == TASK_STATUS__SCAN_HISTORY) || (status == TASK_STATUS__STREAM_SCAN_HISTORY)));
+
+  if (level == TASK_LEVEL__SOURCE) {
    return doStartScanHistoryTask(pTask);
-    } else {
-      ASSERT(pTask->status.taskStatus == TASK_STATUS__NORMAL);
-      stDebug("s-task:%s no need to scan-history data, status:%s, sched-status:%d, ver:%" PRId64, pTask->id.idStr,
-              streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus,
-              walReaderGetCurrentVer(pTask->exec.pWalReader));
-      streamTaskEnablePause(pTask);
-    }
-  } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
+  } else if (level == TASK_LEVEL__AGG) {
    if (pTask->info.fillHistory) {
      streamSetParamForScanHistory(pTask);
      streamTaskEnablePause(pTask);
    }
-  } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
-    if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+  } else if (level == TASK_LEVEL__SINK) {
    stDebug("s-task:%s sink task do nothing to handle scan-history", pTask->id.idStr);
  }
-  }
+
  return 0;
 }

 // check status
-static int32_t doCheckDownstreamStatus(SStreamTask* pTask) {
+void streamTaskCheckDownstream(SStreamTask* pTask) {
  SDataRange*  pRange = &pTask->dataRange;
  STimeWindow* pWindow = &pRange->window;

@ -152,6 +126,8 @@ static int32_t doCheckDownstreamStatus(SStreamTask* pTask) {
      .stage = pTask->pMeta->stage,
  };

+  ASSERT(pTask->status.downstreamReady == 0);
+
  // serialize streamProcessScanHistoryFinishRsp
  if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) {
    req.reqId = tGenIdPI64();
@ -187,14 +163,8 @@ static int32_t doCheckDownstreamStatus(SStreamTask* pTask) {
    }
  } else {
    stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", pTask->id.idStr, pTask->info.nodeId);
-
-    streamTaskSetReady(pTask, 0);
-    streamTaskSetRangeStreamCalc(pTask);
-    streamTaskStartScanHistory(pTask);
-    streamLaunchFillHistoryTask(pTask);
+    doProcessDownstreamReadyRsp(pTask);
  }
-
-  return 0;
 }

 static STaskRecheckInfo* createRecheckInfo(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
@ -280,56 +250,88 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
  }

  if (pInfo->stage != stage) {
-    return TASK_SELF_NEW_STAGE;
+    return TASK_UPSTREAM_NEW_STAGE;
  } else if (pTask->status.downstreamReady != 1) {
+    stDebug("s-task:%s vgId:%d leader:%d, downstream not ready", id, vgId, (pTask->pMeta->role == NODE_ROLE_LEADER));
    return TASK_DOWNSTREAM_NOT_READY;
  } else {
    return TASK_DOWNSTREAM_READY;
  }
 }

-static void doProcessDownstreamReadyRsp(SStreamTask* pTask, int32_t numOfReqs) {
-  streamTaskSetReady(pTask, numOfReqs);
+int32_t onNormalTaskReady(SStreamTask* pTask) {
  const char* id = pTask->id.idStr;

-  int8_t      status = pTask->status.taskStatus;
-  const char* str = streamGetTaskStatusStr(status);
-
-  ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__NORMAL);
+  streamTaskSetReady(pTask);
  streamTaskSetRangeStreamCalc(pTask);

-  if (status == TASK_STATUS__SCAN_HISTORY) {
-    stDebug("s-task:%s enter into scan-history data stage, status:%s", id, str);
-    streamTaskStartScanHistory(pTask);
-  } else {
-    // fill-history tasks are not allowed to reach here.
-    if (pTask->info.fillHistory == 1) {
-      stDebug("s-task:%s fill-history is set normal when start it, try to remove it,set it task to be dropping", id);
-      pTask->status.taskStatus = TASK_STATUS__DROPPING;
-      ASSERT(pTask->hTaskInfo.id.taskId == 0);
-    } else {
-      stDebug("s-task:%s downstream tasks are ready, now ready for data from wal, status:%s", id, str);
-      streamTaskEnablePause(pTask);
-    }
+  char* p = NULL;
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
+  ASSERT(status == TASK_STATUS__READY);
+
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    int64_t startVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
+    if (startVer == -1) {
+      startVer = pTask->chkInfo.nextProcessVer;
    }

-  // when current stream task is ready, check the related fill history task.
+    stDebug("s-task:%s no need to scan-history data, status:%s, sched-status:%d, ready for data from wal ver:%" PRId64,
+            id, p, pTask->status.schedStatus, startVer);
+  } else {
+    stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p, pTask->status.schedStatus);
+  }
+
+  streamTaskEnablePause(pTask);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t onScanhistoryTaskReady(SStreamTask* pTask) {
+  const char* id = pTask->id.idStr;
+
+  // set the state to be ready
+  streamTaskSetReady(pTask);
+  streamTaskSetRangeStreamCalc(pTask);
+
+  char* p = NULL;
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
+  ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY);
+
+  stDebug("s-task:%s enter into scan-history data stage, status:%s", id, p);
+  streamTaskStartScanHistory(pTask);
+
+  // start the related fill-history task, when current task is ready
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
    streamLaunchFillHistoryTask(pTask);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void doProcessDownstreamReadyRsp(SStreamTask* pTask) {
+  EStreamTaskEvent event;
+  if (pTask->info.fillHistory == 0) {
+    event = HAS_RELATED_FILLHISTORY_TASK(pTask)? TASK_EVENT_INIT_STREAM_SCANHIST:TASK_EVENT_INIT;
+  } else {
+    event = TASK_EVENT_INIT_SCANHIST;
+  }
+
+  streamTaskOnHandleEventSuccess(pTask->status.pSM, event);
 }

 int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
  ASSERT(pTask->id.taskId == pRsp->upstreamTaskId);
  const char* id = pTask->id.idStr;
+  int32_t vgId = pTask->pMeta->vgId;

-  if (streamTaskShouldStop(&pTask->status)) {
+  if (streamTaskShouldStop(pTask)) {
    stDebug("s-task:%s should stop, do not do check downstream again", id);
    return TSDB_CODE_SUCCESS;
  }

  if (pRsp->status == TASK_DOWNSTREAM_READY) {
    if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
-      bool found = false;

+      bool found = false;
      int32_t numOfReqs = taosArrayGetSize(pTask->checkReqIds);
      for (int32_t i = 0; i < numOfReqs; i++) {
        int64_t reqId = *(int64_t*)taosArrayGet(pTask->checkReqIds, i);
@ -350,7 +352,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs
        taosArrayDestroy(pTask->checkReqIds);
        pTask->checkReqIds = NULL;

-        doProcessDownstreamReadyRsp(pTask, numOfReqs);
+        doProcessDownstreamReadyRsp(pTask);
      } else {
        int32_t total = taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos);
        stDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id,
@ -362,27 +364,49 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs
        return -1;
      }

-      doProcessDownstreamReadyRsp(pTask, 1);
+      doProcessDownstreamReadyRsp(pTask);
    }
  } else {  // not ready, wait for 100ms and retry
+    if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) {
+      stError(
+          "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, "
+          "not check wait for downstream task nodeUpdate, and all tasks restart",
+          id, pRsp->upstreamNodeId, pRsp->oldStage, (int32_t)pTask->pMeta->stage);
+    } else {
      if (pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) {
        stError(
-          "s-task:%s downstream taskId:0x%x (vgId:%d) vnode-transfer/leader-change detected, not send check again, "
-          "roll-back needed",
+            "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check "
+            "downstream again, nodeUpdate needed",
            id, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
-    } else if (pRsp->status == TASK_SELF_NEW_STAGE) {
-      stError(
-          "s-task:%s vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, not send check "
-          "again, roll-back needed",
-          id, pRsp->oldStage, (int32_t)pTask->pMeta->stage);
-    } else {
+
+        taosThreadMutexLock(&pTask->lock);
+        int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList);
+        bool existed = false;
+        for (int i = 0; i < num; ++i) {
+          SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, i);
+          if (p->nodeId == pRsp->downstreamNodeId) {
+            existed = true;
+            break;
+          }
+        }
+
+        if (!existed) {
+          SDownstreamTaskEpset t = {.nodeId = pRsp->downstreamNodeId};
+          taosArrayPush(pTask->outputInfo.pDownstreamUpdateList, &t);
+          stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", id, vgId,
+                 t.nodeId, (int32_t)taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList));
+        }
+
+        taosThreadMutexUnlock(&pTask->lock);
+        return 0;
+      }
+
      STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp);

      int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
      stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%d, retry in 100ms, ref:%d ", id,
              pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->oldStage, ref);
-
-      taosTmrReset(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer, &pInfo->checkTimer);
+      pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer);
    }
  }

@ -426,30 +450,6 @@ int32_t streamRestoreParam(SStreamTask* pTask) {
  return qRestoreStreamOperatorOption(pTask->exec.pExecutor);
 }

-int32_t streamSetStatusNormal(SStreamTask* pTask) {
-  int32_t status = atomic_load_8(&pTask->status.taskStatus);
-  if (status == TASK_STATUS__DROPPING) {
-    stError("s-task:%s cannot be set normal, since in dropping state", pTask->id.idStr);
-    return -1;
-  } else {
-    stDebug("s-task:%s set task status to be normal, prev:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
-    return 0;
-  }
-}
-
-int32_t streamSetStatusUnint(SStreamTask* pTask) {
-  int32_t status = atomic_load_8(&pTask->status.taskStatus);
-  if (status == TASK_STATUS__DROPPING) {
-    stError("s-task:%s cannot be set uninit, since in dropping state", pTask->id.idStr);
-    return -1;
-  } else {
-    stDebug("s-task:%s set task status to be uninit, prev:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__UNINIT);
-    return 0;
-  }
-}
-
 // source
 int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange *pVerRange, STimeWindow* pWindow) {
  return qStreamSourceScanParamForHistoryScanStep1(pTask->exec.pExecutor, pVerRange, pWindow);
@ -515,9 +515,13 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory
  int32_t taskLevel = pTask->info.taskLevel;
  ASSERT(taskLevel == TASK_LEVEL__AGG || taskLevel == TASK_LEVEL__SINK);

-  if (pTask->status.taskStatus != TASK_STATUS__SCAN_HISTORY) {
+  const char* id = pTask->id.idStr;
+  char*       p = NULL;
+  ETaskStatus status = streamTaskGetStatus(pTask, &p);
+
+  if (status != TASK_STATUS__SCAN_HISTORY && status != TASK_STATUS__STREAM_SCAN_HISTORY) {
    stError("s-task:%s not in scan-history status, status:%s return upstream:0x%x scan-history finish directly",
-           pTask->id.idStr, streamGetTaskStatusStr(pTask->status.taskStatus), pReq->upstreamTaskId);
+           id, p, pReq->upstreamTaskId);

    void*   pBuf = NULL;
    int32_t len = 0;
@ -527,8 +531,8 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory
    initRpcMsg(&msg, 0, pBuf, sizeof(SMsgHead) + len);

    tmsgSendRsp(&msg);
-    stDebug("s-task:%s level:%d notify upstream:0x%x(vgId:%d) to continue process data in WAL", pTask->id.idStr,
-           pTask->info.taskLevel, pReq->upstreamTaskId, pReq->upstreamNodeId);
+    stDebug("s-task:%s level:%d notify upstream:0x%x(vgId:%d) to continue process data in WAL", id,
+            taskLevel, pReq->upstreamTaskId, pReq->upstreamNodeId);
    return 0;
  }

@ -540,13 +544,15 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory

  if (left == 0) {
    int32_t numOfTasks = taosArrayGetSize(pTask->upstreamInfo.pList);
+    if (taskLevel == TASK_LEVEL__AGG) {
      stDebug(
-        "s-task:%s all %d upstream tasks finish scan-history data, set param for agg task for stream data and send "
-        "rsp to all upstream tasks",
-        pTask->id.idStr, numOfTasks);
-
-    if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
+          "s-task:%s all %d upstream tasks finish scan-history data, set param for agg task for stream data processing "
+          "and send rsp to all upstream tasks",
+          id, numOfTasks);
      streamAggUpstreamScanHistoryFinish(pTask);
+    } else {
+      stDebug("s-task:%s all %d upstream task(s) finish scan-history data, and rsp to all upstream tasks", id,
+              numOfTasks);
    }

    // all upstream tasks have completed the scan-history task in the stream time window, let's start to extract data
@ -557,26 +563,25 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory
    if (taskLevel == TASK_LEVEL__AGG) {
      /*int32_t code = */streamTaskScanHistoryDataComplete(pTask);
    } else {  // for sink task, set normal
-      if (pTask->status.taskStatus != TASK_STATUS__PAUSE && pTask->status.taskStatus != TASK_STATUS__STOP &&
-          pTask->status.taskStatus != TASK_STATUS__DROPPING) {
-        streamSetStatusNormal(pTask);
-      }
+      streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
    }
  } else {
    stDebug("s-task:%s receive scan-history data finish msg from upstream:0x%x(index:%d), unfinished:%d",
-           pTask->id.idStr, pReq->upstreamTaskId, pReq->childId, left);
+           id, pReq->upstreamTaskId, pReq->childId, left);
  }

  return 0;
 }

 int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) {
-  ASSERT(pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY);
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
+
+  ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY);
  SStreamMeta* pMeta = pTask->pMeta;

  // execute in the scan history complete call back msg, ready to process data from inputQ
-  streamSetStatusNormal(pTask);
-  streamTaskSetSchedStatusInActive(pTask);
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
+  streamTaskSetSchedStatusInactive(pTask);

  taosWLockLatch(&pMeta->lock);
  streamMetaSaveTask(pMeta, pTask);
@ -604,15 +609,15 @@ static void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask)

  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
    stDebug("s-task:%s set the launch condition for fill-history s-task:%s, window:%" PRId64 " - %" PRId64
-           " ver range:%" PRId64 " - %" PRId64", init:%"PRId64,
+           " verRange:%" PRId64 " - %" PRId64", init:%"PRId64,
           pTask->id.idStr, pHTask->id.idStr, pRange->window.skey, pRange->window.ekey,
           pRange->range.minVer, pRange->range.maxVer, pHTask->execInfo.init);
  } else {
-    stDebug("s-task:%s no fill history condition for non-source task:%s", pTask->id.idStr, pHTask->id.idStr);
+    stDebug("s-task:%s no fill-history condition for non-source task:%s", pTask->id.idStr, pHTask->id.idStr);
  }

  // check if downstream tasks have been ready
-  doCheckDownstreamStatus(pHTask);
+  streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT_SCANHIST);
 }

 static void tryLaunchHistoryTask(void* param, void* tmrId) {
@ -624,12 +629,12 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) {
  if (ppTask) {
    ASSERT((*ppTask)->status.timerActive >= 1);

-    if (streamTaskShouldStop(&(*ppTask)->status)) {
-      const char* pStatus = streamGetTaskStatusStr((*ppTask)->status.taskStatus);
-
+    if (streamTaskShouldStop(*ppTask)) {
+      char* p = NULL;
+      streamTaskGetStatus((*ppTask), &p);
      int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1);
      stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d",
-              (*ppTask)->id.idStr, pStatus, (*ppTask)->hTaskInfo.retryTimes, ref);
+              (*ppTask)->id.idStr, p, (*ppTask)->hTaskInfo.retryTimes, ref);

      taosMemoryFree(pInfo);
      taosWUnLockLatch(&pMeta->lock);
@ -665,9 +670,11 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) {

      // abort the timer if intend to stop task
      SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId);
-      if (pHTask == NULL && (!streamTaskShouldStop(&pTask->status))) {
-        const char* p = streamGetTaskStatusStr(pTask->status.taskStatus);
+      if (pHTask == NULL && (!streamTaskShouldStop(pTask))) {
+        char*   p = NULL;
        int32_t hTaskId = pHTaskInfo->id.taskId;
+
+        streamTaskGetStatus(pTask, &p);
        stDebug(
            "s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d",
            pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes);
@ -713,11 +720,8 @@ SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, int64_t streamId, in
 int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
  SStreamMeta* pMeta = pTask->pMeta;
  int32_t      hTaskId = pTask->hTaskInfo.id.taskId;
-  if (hTaskId == 0) {
-    return TSDB_CODE_SUCCESS;
-  }

-  ASSERT(pTask->status.downstreamReady == 1);
+  ASSERT((hTaskId != 0) && (pTask->status.downstreamReady == 1));
  stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", pTask->id.idStr,
          pTask->hTaskInfo.id.streamId, hTaskId);

@ -765,7 +769,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
 }

 int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask) {
-  if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__DROPPING) {
    return 0;
  }

@ -897,7 +901,7 @@ int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistory
 void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
  SDataRange* pRange = &pTask->dataRange;

-  if (pTask->hTaskInfo.id.taskId == 0) {
+  if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) {
    if (pTask->info.fillHistory == 1) {
      stDebug("s-task:%s fill-history task, time window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 "-%" PRId64,
              pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer);
@ -929,21 +933,8 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
  }
 }

-// only the downstream tasks are ready, set the task to be ready to work.
-void streamTaskCheckDownstream(SStreamTask* pTask) {
-  if (pTask->info.fillHistory) {
-    stDebug("s-task:%s fill history task, wait for being launched", pTask->id.idStr);
-    return;
-  }
-
-  ASSERT(pTask->status.downstreamReady == 0);
-  doCheckDownstreamStatus(pTask);
-}
-
-// normal -> pause, pause/stop/dropping -> pause, halt -> pause, scan-history -> pause
 void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta) {
-  int64_t st = taosGetTimestampMs();
-
+#if 0
  int8_t status = pTask->status.taskStatus;
  if (status == TASK_STATUS__DROPPING) {
    stDebug("vgId:%d s-task:%s task already dropped, do nothing", pMeta->vgId, pTask->id.idStr);
@ -1001,43 +992,54 @@ void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta) {
  stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
  taosWUnLockLatch(&pMeta->lock);

+#endif
+
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_PAUSE);
+
+  int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
+  stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
+
  // in case of fill-history task, stop the tsdb file scan operation.
  if (pTask->info.fillHistory == 1) {
    void* pExecutor = pTask->exec.pExecutor;
    qKillTask(pExecutor, TSDB_CODE_SUCCESS);
  }

-  int64_t el = taosGetTimestampMs() - st;
-  stDebug("vgId:%d s-task:%s set pause flag, prev:%s, pause elapsed time:%dms", pMeta->vgId, pTask->id.idStr,
-         streamGetTaskStatusStr(pTask->status.keepTaskStatus), (int32_t)el);
+  stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr);
 }

-void streamTaskResume(SStreamTask* pTask, SStreamMeta* pMeta) {
-  int8_t status = pTask->status.taskStatus;
+void streamTaskResume(SStreamTask* pTask) {
+  char*        p = NULL;
+  ETaskStatus  status = streamTaskGetStatus(pTask, &p);
+  SStreamMeta* pMeta = pTask->pMeta;
+
+  if (status == TASK_STATUS__PAUSE || status == TASK_STATUS__HALT) {
+    streamTaskRestoreStatus(pTask);
+
+    char* pNew = NULL;
+    streamTaskGetStatus(pTask, &pNew);
    if (status == TASK_STATUS__PAUSE) {
-    pTask->status.taskStatus = pTask->status.keepTaskStatus;
-    pTask->status.keepTaskStatus = TASK_STATUS__NORMAL;
      int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
-    stInfo("vgId:%d s-task:%s resume from pause, status:%s. pause task num:%d", pMeta->vgId, pTask->id.idStr, streamGetTaskStatusStr(status), num);
-  } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
-    int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
-    stInfo("vgId:%d s-task:%s sink task.resume from pause, status:%s. pause task num:%d", pMeta->vgId, pTask->id.idStr, streamGetTaskStatusStr(status), num);
+      stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, p, num);
    } else {
-    stError("s-task:%s not in pause, failed to resume, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
+      stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, p);
+    }
+  } else {
+    stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, p);
  }
 }

 // todo fix race condition
 void streamTaskDisablePause(SStreamTask* pTask) {
  // pre-condition check
-  const char* id = pTask->id.idStr;
-  while (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-    stDebug("s-task:%s already in pause, wait for pause being cancelled, and set pause disabled, recheck in 100ms", id);
-    taosMsleep(100);
-  }
-
-  stDebug("s-task:%s disable task pause", id);
-  pTask->status.pauseAllowed = 0;
+//  const char* id = pTask->id.idStr;
+//  while (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+//    stDebug("s-task:%s already in pause, wait for pause being cancelled, and set pause disabled, recheck in 100ms", id);
+//    taosMsleep(100);
+//  }
+//
+//  stDebug("s-task:%s disable task pause", id);
+//  pTask->status.pauseAllowed = 0;
 }

 void streamTaskEnablePause(SStreamTask* pTask) {
@ -1045,15 +1047,30 @@ void streamTaskEnablePause(SStreamTask* pTask) {
  pTask->status.pauseAllowed = 1;
 }

-void streamTaskResumeFromHalt(SStreamTask* pTask) {
-  const char* id = pTask->id.idStr;
-  int8_t status = pTask->status.taskStatus;
-  if (status != TASK_STATUS__HALT) {
-    stError("s-task:%s not in halt status, status:%s", id, streamGetTaskStatusStr(status));
-    return;
+int32_t streamMetaUpdateTaskReadyInfo(SStreamTask* pTask) {
+  SStreamMeta* pMeta = pTask->pMeta;
+
+  taosWLockLatch(&pMeta->lock);
+
+  STaskId id = streamTaskExtractKey(pTask);
+  taosHashPut(pMeta->startInfo.pReadyTaskSet, &id, sizeof(id), NULL, 0);
+
+  int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta);
+
+  if (taosHashGetSize(pMeta->startInfo.pReadyTaskSet) == numOfTotal) {
+    STaskStartInfo* pStartInfo = &pMeta->startInfo;
+
+    pStartInfo->readyTs = pTask->execInfo.start;
+    pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0;
+
+    streamMetaResetStartInfo(pStartInfo);
+
+    stDebug("vgId:%d all %d task(s) are started successfully, last ready task:%s level:%d, startTs:%" PRId64
+            ", readyTs:%" PRId64 " total elapsed time:%.2fs",
+            pMeta->vgId, numOfTotal, pTask->id.idStr, pTask->info.taskLevel, pStartInfo->startTs, pStartInfo->readyTs,
+            pStartInfo->elapsedTime / 1000.0);
  }

-  pTask->status.taskStatus = pTask->status.keepTaskStatus;
-  pTask->status.keepTaskStatus = TASK_STATUS__NORMAL;
-  stDebug("s-task:%s resume from halt, current status:%s", id, streamGetTaskStatusStr(pTask->status.taskStatus));
+  taosWUnLockLatch(&pMeta->lock);
+  return TSDB_CODE_SUCCESS;
 }
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@ -19,6 +19,7 @@
 #include "tstream.h"
 #include "ttimer.h"
 #include "wal.h"
+#include "streamsm.h"

 static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo);

@ -34,8 +35,11 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, bool fillHistory
  SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
  if (pTask == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    stError("s-task:0x%" PRIx64 " failed malloc new stream task, size:%d, code:%s", streamId,
+            (int32_t)sizeof(SStreamTask), tstrerror(terrno));
    return NULL;
  }
+
  pTask->ver = SSTREAM_TASK_VER;
  pTask->id.taskId = tGenIdPI32();
  pTask->id.streamId = streamId;
@ -43,13 +47,19 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, bool fillHistory
  pTask->info.fillHistory = fillHistory;
  pTask->info.triggerParam = triggerParam;

+  pTask->status.pSM = streamCreateStateMachine(pTask);
+  if (pTask->status.pSM == NULL) {
+    taosMemoryFreeClear(pTask);
+    return NULL;
+  }
+
  char buf[128] = {0};
  sprintf(buf, "0x%" PRIx64 "-%d", pTask->id.streamId, pTask->id.taskId);

  pTask->id.idStr = taosStrdup(buf);
  pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
-  pTask->status.taskStatus = (fillHistory || hasFillhistory)? TASK_STATUS__SCAN_HISTORY:TASK_STATUS__NORMAL;
-  pTask->inputInfo.status = TASK_INPUT_STATUS__NORMAL;
+  pTask->status.taskStatus = (fillHistory || hasFillhistory) ? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY;
+  pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
  pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;

  if (fillHistory) {
@ -296,9 +306,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
  int32_t taskId = pTask->id.taskId;

  STaskExecStatisInfo* pStatis = &pTask->execInfo;
-
-  stDebug("start to free s-task:0x%x, %p, state:%p, status:%s", taskId, pTask, pTask->pState,
-         streamGetTaskStatusStr(pTask->status.taskStatus));
+  stDebug("start to free s-task:0x%x, %p, state:%p", taskId, pTask, pTask->pState);

  stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64
         ", updateCount:%d latestUpdate:%" PRId64 ", latestCheckPoint:%" PRId64 ", ver:%" PRId64
@ -309,8 +317,8 @@ void tFreeStreamTask(SStreamTask* pTask) {

  // remove the ref by timer
  while (pTask->status.timerActive > 0) {
-    stDebug("s-task:%s wait for task stop timer activities", pTask->id.idStr);
-    taosMsleep(10);
+    stDebug("s-task:%s wait for task stop timer activities, ref:%d", pTask->id.idStr, pTask->status.timerActive);
+    taosMsleep(100);
  }

  if (pTask->schedInfo.pTimer != NULL) {
@ -329,8 +337,8 @@ void tFreeStreamTask(SStreamTask* pTask) {
  }

  int32_t status = atomic_load_8((int8_t*)&(pTask->status.taskStatus));
-  if (pTask->inputInfo.queue) {
-    streamQueueClose(pTask->inputInfo.queue, pTask->id.taskId);
+  if (pTask->inputq.queue) {
+    streamQueueClose(pTask->inputq.queue, pTask->id.taskId);
  }

  if (pTask->outputq.queue) {
@ -384,13 +392,18 @@ void tFreeStreamTask(SStreamTask* pTask) {
    pTask->pRspMsgList = NULL;
  }

+  pTask->status.pSM = streamDestroyStateMachine(pTask->status.pSM);
+
  streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo);

  pTask->msgInfo.pRetryList = taosArrayDestroy(pTask->msgInfo.pRetryList);
  taosMemoryFree(pTask->outputInfo.pTokenBucket);
  taosThreadMutexDestroy(&pTask->lock);
-  taosMemoryFree(pTask);

+  taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList);
+  pTask->outputInfo.pDownstreamUpdateList = NULL;
+
+  taosMemoryFree(pTask);
  stDebug("s-task:0x%x free task completed", taskId);
 }

@ -399,16 +412,23 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
  pTask->refCnt = 1;
  pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
  pTask->status.timerActive = 0;
-  pTask->inputInfo.queue = streamQueueOpen(512 << 10);
+  pTask->inputq.queue = streamQueueOpen(512 << 10);
  pTask->outputq.queue = streamQueueOpen(512 << 10);

-  if (pTask->inputInfo.queue == NULL || pTask->outputq.queue == NULL) {
+  if (pTask->inputq.queue == NULL || pTask->outputq.queue == NULL) {
    stError("s-task:%s failed to prepare the input/output queue, initialize task failed", pTask->id.idStr);
    return TSDB_CODE_OUT_OF_MEMORY;
  }

+  pTask->status.pSM = streamCreateStateMachine(pTask);
+  if (pTask->status.pSM == NULL) {
+    stError("s-task:%s failed create state-machine for stream task, initialization failed, code:%s", pTask->id.idStr,
+        tstrerror(terrno));
+    return terrno;
+  }
+
  pTask->execInfo.created = taosGetTimestampMs();
-  pTask->inputInfo.status = TASK_INPUT_STATUS__NORMAL;
+  pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
  pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
  pTask->pMeta = pMeta;

@ -427,7 +447,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i

  // 2MiB per second for sink task
  // 50 times sink operator per second
-  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, 2);
+  streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate);

  TdThreadMutexAttr attr = {0};
  int code = taosThreadMutexAttrInit(&attr);
@ -445,6 +465,11 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
  taosThreadMutexInit(&pTask->lock, &attr);
  streamTaskOpenAllUpstreamInput(pTask);

+  pTask->outputInfo.pDownstreamUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset));
+  if (pTask->outputInfo.pDownstreamUpdateList == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
  return TSDB_CODE_SUCCESS;
 }

@ -453,7 +478,9 @@ int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) {
    return 0;
  } else {
    int32_t type = pTask->outputInfo.type;
-    if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__TABLE) {
+    if (type == TASK_OUTPUT__TABLE) {
+      return 0;
+    } else if (type == TASK_OUTPUT__FIXED_DISPATCH) {
      return 1;
    } else {
      SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
@ -562,13 +589,7 @@ int32_t streamTaskStop(SStreamTask* pTask) {
  int64_t      st = taosGetTimestampMs();
  const char*  id = pTask->id.idStr;

-  taosThreadMutexLock(&pTask->lock);
-  if (pTask->status.taskStatus == TASK_STATUS__CK) {
-    stDebug("s-task:%s in checkpoint will be discarded since task is stopped", id);
-  }
-  pTask->status.taskStatus = TASK_STATUS__STOP;
-  taosThreadMutexUnlock(&pTask->lock);
-
+  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP);
  qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
  while (/*pTask->status.schedStatus != TASK_SCHED_STATUS__INACTIVE */ !streamTaskIsIdle(pTask)) {
    stDebug("s-task:%s level:%d wait for task to be idle and then close, check again in 100ms", id,
@ -612,7 +633,7 @@ int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) {

  p->latestUpdateTs = taosGetTimestampMs();
  p->updateCount += 1;
-  stDebug("s-task:%s update task nodeEp epset, updatedNodes:%d, updateCount:%d, prevTs:%" PRId64, pTask->id.idStr,
+  stDebug("s-task:0x%x update task nodeEp epset, updatedNodes:%d, updateCount:%d, prevTs:%" PRId64, pTask->id.taskId,
         numOfNodes, p->updateCount, prevTs);

  for (int32_t i = 0; i < taosArrayGetSize(pNodeList); ++i) {
@ -667,7 +688,7 @@ int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) {
  return status;
 }

-int8_t streamTaskSetSchedStatusInActive(SStreamTask* pTask) {
+int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
  taosThreadMutexLock(&pTask->lock);
  int8_t status = pTask->status.schedStatus;
  ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE ||
@ -678,6 +699,25 @@ int8_t streamTaskSetSchedStatusInActive(SStreamTask* pTask) {
  return status;
 }

+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask) {
+  SStreamMeta* pMeta = pTask->pMeta;
+  if (pTask->info.fillHistory == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId};
+  SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId));
+
+  if (ppStreamTask != NULL) {
+    CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask));
+    streamMetaSaveTask(pMeta, *ppStreamTask);
+    stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr,
+            (int32_t)sTaskId.taskId);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId) {
  SVDropStreamTaskReq *pReq = rpcMallocCont(sizeof(SVDropStreamTaskReq));
  if (pReq == NULL) {
@ -696,7 +736,7 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
    return code;
  }

-  stDebug("vgId:%d build and send drop table:0x%x msg", vgId, pTaskId->taskId);
+  stDebug("vgId:%d build and send drop task:0x%x msg", vgId, pTaskId->taskId);
  return code;
 }

@ -719,20 +759,6 @@ void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo) {
  pInfo->retryTimes += 1;
 }

-const char* streamGetTaskStatusStr(int32_t status) {
-  switch(status) {
-    case TASK_STATUS__NORMAL: return "normal";
-    case TASK_STATUS__SCAN_HISTORY: return "scan-history";
-    case TASK_STATUS__HALT: return "halt";
-    case TASK_STATUS__PAUSE: return "paused";
-    case TASK_STATUS__CK: return "check-point";
-    case TASK_STATUS__DROPPING: return "dropping";
-    case TASK_STATUS__STOP: return "stop";
-    case TASK_STATUS__UNINIT: return "uninitialized";
-    default:return "";
-  }
-}
-
 void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask) {
  pEntry->id.streamId = pTask->id.streamId;
  pEntry->id.taskId = pTask->id.taskId;
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@ -0,0 +1,529 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "streamInt.h"
+#include "streamsm.h"
+#include "tmisce.h"
+#include "tstream.h"
+#include "ttimer.h"
+#include "wal.h"
+
+SStreamTaskState StreamTaskStatusList[9] = {
+    {.state = TASK_STATUS__READY, .name = "ready"},
+    {.state = TASK_STATUS__DROPPING, .name = "dropped"},
+    {.state = TASK_STATUS__UNINIT, .name = "uninit"},
+    {.state = TASK_STATUS__STOP, .name = "stop"},
+    {.state = TASK_STATUS__SCAN_HISTORY, .name = "scan-history"},
+    {.state = TASK_STATUS__HALT, .name = "halt"},
+    {.state = TASK_STATUS__PAUSE, .name = "paused"},
+    {.state = TASK_STATUS__CK, .name = "checkpoint"},
+    {.state = TASK_STATUS__STREAM_SCAN_HISTORY, .name = "stream-scan-history"},
+};
+
+SStreamEventInfo StreamTaskEventList[12] = {
+    {.event = 0, .name = ""},  // dummy event, place holder
+    {.event = TASK_EVENT_INIT, .name = "initialize"},
+    {.event = TASK_EVENT_INIT_SCANHIST, .name = "scan-history-init"},
+    {.event = TASK_EVENT_INIT_STREAM_SCANHIST, .name = "stream-scan-history-init"},
+    {.event = TASK_EVENT_SCANHIST_DONE, .name = "scan-history-completed"},
+    {.event = TASK_EVENT_STOP, .name = "stopping"},
+    {.event = TASK_EVENT_GEN_CHECKPOINT, .name = "checkpoint"},
+    {.event = TASK_EVENT_CHECKPOINT_DONE, .name = "checkpoint-done"},
+    {.event = TASK_EVENT_PAUSE, .name = "pausing"},
+    {.event = TASK_EVENT_RESUME, .name = "resuming"},
+    {.event = TASK_EVENT_HALT, .name = "halting"},
+    {.event = TASK_EVENT_DROPPING, .name = "dropping"},
+};
+
+static TdThreadOnce streamTaskStateMachineInit = PTHREAD_ONCE_INIT;
+static SArray*      streamTaskSMTrans = NULL;
+
+static int32_t streamTaskInitStatus(SStreamTask* pTask);
+static int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask);
+static int32_t initStateTransferTable();
+static void    doInitStateTransferTable(void);
+
+static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event,
+                                            __state_trans_fn fn, __state_trans_succ_fn succFn,
+                                            SAttachedEventInfo* pEventInfo, bool autoInvoke);
+
+static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; }
+
+static int32_t attachEvent(SStreamTask* pTask, SAttachedEventInfo* pEvtInfo) {
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
+  stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p,
+          StreamTaskEventList[pEvtInfo->event].name, StreamTaskStatusList[pEvtInfo->status].name);
+  taosArrayPush(pTask->status.pSM->pWaitingEventList, pEvtInfo);
+  return 0;
+}
+
+int32_t streamTaskInitStatus(SStreamTask* pTask) {
+  pTask->execInfo.init = taosGetTimestampMs();
+
+  stDebug("s-task:%s start init, and check downstream tasks, set the init ts:%" PRId64, pTask->id.idStr,
+          pTask->execInfo.init);
+  streamTaskCheckDownstream(pTask);
+  return 0;
+}
+
+int32_t streamTaskSetReadyForWal(SStreamTask* pTask) {
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    stDebug("s-task:%s ready for extract data from wal", pTask->id.idStr);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t streamTaskDoCheckpoint(SStreamTask* pTask) {
+  stDebug("s-task:%s start to do checkpoint", pTask->id.idStr);
+  return 0;
+}
+
+int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) {
+  ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));
+
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    pTask->hTaskInfo.haltVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
+    if (pTask->hTaskInfo.haltVer == -1) {
+      pTask->hTaskInfo.haltVer = pTask->dataRange.range.maxVer + 1;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+// todo optimize the perf of find the trans objs by using hash table
+static STaskStateTrans* streamTaskFindTransform(ETaskStatus state, const EStreamTaskEvent event) {
+  int32_t numOfTrans = taosArrayGetSize(streamTaskSMTrans);
+  for (int32_t i = 0; i < numOfTrans; ++i) {
+    STaskStateTrans* pTrans = taosArrayGet(streamTaskSMTrans, i);
+    if (pTrans->state.state == state && pTrans->event == event) {
+      return pTrans;
+    }
+  }
+
+  if (event == TASK_EVENT_CHECKPOINT_DONE && state == TASK_STATUS__STOP) {
+
+  } else if (event == TASK_EVENT_GEN_CHECKPOINT && state == TASK_STATUS__UNINIT) {
+    // the task is set to uninit due to nodeEpset update, during processing checkpoint-trigger block.
+  } else {
+    ASSERT(0);
+  }
+
+  return NULL;
+}
+
+void streamTaskRestoreStatus(SStreamTask* pTask) {
+  SStreamTaskSM* pSM = pTask->status.pSM;
+  taosThreadMutexLock(&pTask->lock);
+  ASSERT(pSM->pActiveTrans == NULL);
+
+  ASSERT(pSM->current.state == TASK_STATUS__PAUSE || pSM->current.state == TASK_STATUS__HALT);
+
+  SStreamTaskState state = pSM->current;
+  pSM->current = pSM->prev.state;
+
+  pSM->prev.state = state;
+  pSM->prev.evt = 0;
+
+  pSM->startTs = taosGetTimestampMs();
+  stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name);
+
+  taosThreadMutexUnlock(&pTask->lock);
+}
+
+SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) {
+  initStateTransferTable();
+  const char* id = pTask->id.idStr;
+
+  SStreamTaskSM* pSM = taosMemoryCalloc(1, sizeof(SStreamTaskSM));
+  if (pSM == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM),
+            tstrerror(terrno));
+    return NULL;
+  }
+
+  pSM->pTask = pTask;
+  pSM->pWaitingEventList = taosArrayInit(4, sizeof(SAttachedEventInfo));
+  if (pSM->pWaitingEventList == NULL) {
+    taosMemoryFree(pSM);
+
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM),
+            tstrerror(terrno));
+    return NULL;
+  }
+
+  // set the initial state for the state-machine of stream task
+  pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
+  pSM->startTs = taosGetTimestampMs();
+  return pSM;
+}
+
+void* streamDestroyStateMachine(SStreamTaskSM* pSM) {
+  if (pSM == NULL) {
+    return NULL;
+  }
+
+  taosArrayDestroy(pSM->pWaitingEventList);
+  taosMemoryFree(pSM);
+  return NULL;
+}
+
+static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans) {
+  SStreamTask* pTask = pSM->pTask;
+  const char*  id = pTask->id.idStr;
+
+  if (pTrans->attachEvent.event != 0) {
+    attachEvent(pTask, &pTrans->attachEvent);
+    taosThreadMutexUnlock(&pTask->lock);
+
+    while (1) {
+      // wait for the task to be here
+      taosThreadMutexLock(&pTask->lock);
+      ETaskStatus s = streamTaskGetStatus(pTask, NULL);
+      taosThreadMutexUnlock(&pTask->lock);
+
+      if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {
+        stDebug("s-task:%s attached event:%s handled", id, StreamTaskEventList[pTrans->event].name);
+        return TSDB_CODE_SUCCESS;
+      } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP) {  // this event has been handled already
+        stDebug("s-task:%s not handle event:%s yet, wait for 100ms and recheck", id, StreamTaskEventList[event].name);
+        taosMsleep(100);
+      } else {
+        stDebug("s-task:%s is dropped or stopped already, not wait.", id);
+        return TSDB_CODE_INVALID_PARA;
+      }
+    }
+
+  } else {  // override current active trans
+    pSM->pActiveTrans = pTrans;
+    pSM->startTs = taosGetTimestampMs();
+    taosThreadMutexUnlock(&pTask->lock);
+
+    int32_t code = pTrans->pAction(pTask);
+    // todo handle error code;
+
+    if (pTrans->autoInvokeEndFn) {
+      streamTaskOnHandleEventSuccess(pSM, event);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
+  SStreamTask*     pTask = pSM->pTask;
+  STaskStateTrans* pTrans = NULL;
+
+  while (1) {
+    taosThreadMutexLock(&pTask->lock);
+    if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
+      taosThreadMutexUnlock(&pTask->lock);
+      taosMsleep(100);
+      stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
+              pTask->id.idStr, pSM->current.name, StreamTaskEventList[pSM->pActiveTrans->event].name);
+    } else {
+      pTrans = streamTaskFindTransform(pSM->current.state, event);
+      if (pTrans == NULL) {
+        stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, StreamTaskEventList[event].name);
+        taosThreadMutexUnlock(&pTask->lock);
+        return TSDB_CODE_INVALID_PARA;  // todo: set new error code// failed to handle the event.
+      }
+
+      if (pSM->pActiveTrans != NULL) {
+        // currently in some state transfer procedure, not auto invoke transfer, abort it
+        stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now",
+                pTask->id.idStr, StreamTaskEventList[pSM->pActiveTrans->event].name, pSM->current.name,
+                pSM->pActiveTrans->next.name, StreamTaskEventList[event].name);
+      }
+
+      doHandleEvent(pSM, event, pTrans);
+      break;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void keepPrevInfo(SStreamTaskSM* pSM) {
+  STaskStateTrans* pTrans = pSM->pActiveTrans;
+
+  pSM->prev.state = pSM->current;
+  pSM->prev.evt = pTrans->event;
+}
+
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event) {
+  SStreamTask* pTask = pSM->pTask;
+
+  // do update the task status
+  taosThreadMutexLock(&pTask->lock);
+  STaskStateTrans* pTrans = pSM->pActiveTrans;
+
+  if (pTrans == NULL) {
+    ETaskStatus s = pSM->current.state;
+    ASSERT(s == TASK_STATUS__DROPPING || s == TASK_STATUS__PAUSE || s == TASK_STATUS__STOP);
+    // the pSM->prev.evt may be 0, so print string is not appropriate.
+    stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", pTask->id.idStr,
+            StreamTaskEventList[event].name, pSM->current.name, StreamTaskEventList[pSM->prev.evt].name);
+
+    taosThreadMutexUnlock(&pTask->lock);
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  if (pTrans->event != event) {
+    stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", pTask->id.idStr,
+           StreamTaskEventList[event].name, pSM->current.name, StreamTaskEventList[pTrans->event].name);
+    taosThreadMutexUnlock(&pTask->lock);
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  keepPrevInfo(pSM);
+
+  pSM->current = pTrans->next;
+  pSM->pActiveTrans = NULL;
+
+  // on success callback, add into lock if necessary, or maybe we should add an option for this?
+  pTrans->pSuccAction(pTask);
+
+  if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
+    int64_t el = (taosGetTimestampMs() - pSM->startTs);
+    stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr,
+            StreamTaskEventList[pTrans->event].name, el, pSM->prev.state.name, pSM->current.name);
+
+    SAttachedEventInfo* pEvtInfo = taosArrayPop(pSM->pWaitingEventList);
+
+    // OK, let's handle the attached event, since the task has reached the required status now
+    if (pSM->current.state == pEvtInfo->status) {
+      stDebug("s-task:%s handle the attached event:%s, state:%s", pTask->id.idStr,
+              StreamTaskEventList[pEvtInfo->event].name, pSM->current.name);
+
+      STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event);
+      ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL);
+
+      pSM->pActiveTrans = pNextTrans;
+      pSM->startTs = taosGetTimestampMs();
+      taosThreadMutexUnlock(&pTask->lock);
+
+      int32_t code = pNextTrans->pAction(pSM->pTask);
+      if (pNextTrans->autoInvokeEndFn) {
+        return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event);
+      } else {
+        return code;
+      }
+    }
+  } else {
+    taosThreadMutexUnlock(&pTask->lock);
+
+    int64_t el = (taosGetTimestampMs() - pSM->startTs);
+    stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr,
+            StreamTaskEventList[pTrans->event].name, el, pSM->prev.state.name, pSM->current.name);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr) {
+  SStreamTaskState s = pTask->status.pSM->current;  // copy one obj in case of multi-thread environment
+  if (pStr != NULL) {
+    *pStr = s.name;
+  }
+  return s.state;
+}
+
+const char* streamTaskGetStatusStr(ETaskStatus status) {
+  return StreamTaskStatusList[status].name;
+}
+
+void streamTaskResetStatus(SStreamTask* pTask) {
+  SStreamTaskSM* pSM = pTask->status.pSM;
+
+  taosThreadMutexLock(&pTask->lock);
+  stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr,
+          pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name);
+
+  pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
+  pSM->pActiveTrans = NULL;
+  taosArrayClear(pSM->pWaitingEventList);
+  taosThreadMutexUnlock(&pTask->lock);
+
+  // clear the downstream ready status
+  pTask->status.downstreamReady = 0;
+}
+
+void streamTaskSetStatusReady(SStreamTask* pTask) {
+  SStreamTaskSM* pSM = pTask->status.pSM;
+  if (pSM->current.state == TASK_STATUS__DROPPING) {
+    stError("s-task:%s task in dropping state, cannot be set ready", pTask->id.idStr);
+    return;
+  }
+
+  taosThreadMutexLock(&pTask->lock);
+
+  pSM->prev.state = pSM->current;
+  pSM->prev.evt = 0;
+
+  pSM->current = StreamTaskStatusList[TASK_STATUS__READY];
+  pSM->startTs = taosGetTimestampMs();
+  pSM->pActiveTrans = NULL;
+  taosArrayClear(pSM->pWaitingEventList);
+
+  taosThreadMutexUnlock(&pTask->lock);
+}
+
+STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn,
+                                     __state_trans_succ_fn succFn, SAttachedEventInfo* pEventInfo, bool autoInvoke) {
+  STaskStateTrans trans = {0};
+  trans.state = StreamTaskStatusList[current];
+  trans.next = StreamTaskStatusList[next];
+  trans.event = event;
+
+  if (pEventInfo != NULL) {
+    trans.attachEvent = *pEventInfo;
+  } else {
+    trans.attachEvent.event = 0;
+    trans.attachEvent.status = 0;
+  }
+
+  trans.pAction = (fn != NULL) ? fn : dummyFn;
+  trans.pSuccAction = (succFn != NULL) ? succFn : dummyFn;
+  trans.autoInvokeEndFn = autoInvoke;
+  return trans;
+}
+
+int32_t initStateTransferTable() {
+  taosThreadOnce(&streamTaskStateMachineInit, doInitStateTransferTable);
+  return TSDB_CODE_SUCCESS;
+}
+
+void doInitStateTransferTable(void) {
+  streamTaskSMTrans = taosArrayInit(8, sizeof(STaskStateTrans));
+
+  // initialization event handle
+  STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT,
+                                               streamTaskInitStatus, onNormalTaskReady, false, false);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST,
+                               streamTaskInitStatus, onScanhistoryTaskReady, false, false);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST,
+                               streamTaskInitStatus, onScanhistoryTaskReady, false, false);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // scan-history related event
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL,
+                               NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL,
+                               NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // halt stream task, from other task status
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL,
+                               streamTaskKeepCurrentVerInWal, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT};
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL,
+                               streamTaskKeepCurrentVerInWal, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal,
+                               &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL,
+                               streamTaskKeepCurrentVerInWal, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // checkpoint related event
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL,
+                               streamTaskDoCheckpoint, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans =
+      createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // pause & resume related event handle
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE};
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // resume is completed by restore status of state-machine
+
+  // stop related event
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+
+  // dropping related event
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+  trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  taosArrayPush(streamTaskSMTrans, &trans);
+}
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@ -249,8 +249,8 @@ int32_t syncNodeOnRequestVote(SSyncNode* pNode, const SRpcMsg* pMsg);
 int32_t syncNodeOnRequestVoteReply(SSyncNode* pNode, const SRpcMsg* pMsg);
 int32_t syncNodeOnAppendEntries(SSyncNode* pNode, const SRpcMsg* pMsg);
 int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pMsg);
-int32_t syncNodeOnSnapshot(SSyncNode* ths, const SRpcMsg* pMsg);
-int32_t syncNodeOnSnapshotRsp(SSyncNode* ths, const SRpcMsg* pMsg);
+int32_t syncNodeOnSnapshot(SSyncNode* ths, SRpcMsg* pMsg);
+int32_t syncNodeOnSnapshotRsp(SSyncNode* ths, SRpcMsg* pMsg);
 int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pMsg);
 int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pMsg);
 int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pMsg);
--- a/source/libs/sync/inc/syncSnapshot.h
+++ b/source/libs/sync/inc/syncSnapshot.h
@ -22,21 +22,41 @@ extern "C" {

 #include "syncInt.h"

-#define SYNC_SNAPSHOT_SEQ_INVALID      -2
 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE  -3
-#define SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT -1
+#define SYNC_SNAPSHOT_SEQ_INVALID      -2
+#define SYNC_SNAPSHOT_SEQ_PREP         -1
 #define SYNC_SNAPSHOT_SEQ_BEGIN        0
 #define SYNC_SNAPSHOT_SEQ_END          0x7FFFFFFF

 #define SYNC_SNAPSHOT_RETRY_MS 5000

+typedef struct SSyncSnapBuffer {
+  void         *entries[TSDB_SYNC_SNAP_BUFFER_SIZE];
+  int64_t       start;
+  int64_t       cursor;
+  int64_t       end;
+  int64_t       size;
+  TdThreadMutex mutex;
+  void (*entryDeleteCb)(void *ptr);
+} SSyncSnapBuffer;
+
+typedef struct SyncSnapBlock {
+  int32_t seq;
+  int8_t  acked;
+  int64_t sendTimeMs;
+
+  int16_t blockType;
+  void   *pBlock;
+  int32_t blockLen;
+} SyncSnapBlock;
+
+void syncSnapBlockDestroy(void *ptr);
+
 typedef struct SSyncSnapshotSender {
  int8_t         start;
  int32_t        seq;
  int32_t        ack;
  void          *pReader;
-  void          *pCurrentBlock;
-  int32_t        blockLen;
  SSnapshotParam snapshotParam;
  SSnapshot      snapshot;
  SSyncCfg       lastConfig;
@ -47,6 +67,9 @@ typedef struct SSyncSnapshotSender {
  int64_t        lastSendTime;
  bool           finish;

+  // ring buffer for ack
+  SSyncSnapBuffer *pSndBuf;
+
  // init when create
  SSyncNode *pSyncNode;
  int32_t    replicaIndex;
@ -72,6 +95,9 @@ typedef struct SSyncSnapshotReceiver {
  SSnapshotParam snapshotParam;
  SSnapshot      snapshot;

+  // buffer
+  SSyncSnapBuffer *pRcvBuf;
+
  // init when create
  SSyncNode *pSyncNode;
 } SSyncSnapshotReceiver;
@ -83,8 +109,8 @@ void                   snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver);
 bool                   snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);

 // on message
-int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg);
-int32_t syncNodeOnSnapshotRsp(SSyncNode *ths, const SRpcMsg *pMsg);
+// int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg);
+// int32_t syncNodeOnSnapshotRsp(SSyncNode *ths, const SRpcMsg *pMsg);

 SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode *pSyncNode, SyncIndex snapshotLastApplyIndex);

--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@ -818,7 +818,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) {

  if (!taosCheckExistFile(pSyncNode->configPath)) {
    // create a new raft config file
-    sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId);
+    sInfo("vgId:%d, create a new raft config file", pSyncInfo->vgId);
+    pSyncNode->vgId = pSyncInfo->vgId;
    pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy;
    pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy;
    pSyncNode->raftCfg.lastConfigIndex = pSyncInfo->syncCfg.lastIndex;
--- a/source/libs/sync/src/syncPipeline.c
+++ b/source/libs/sync/src/syncPipeline.c
@ -797,7 +797,7 @@ _out:
    pMgr->retryBackoff = syncLogReplGetNextRetryBackoff(pMgr);
    SSyncLogBuffer* pBuf = pNode->pLogBuf;
    sInfo("vgId:%d, resend %d sync log entries. dest:%" PRIx64 ", indexes:%" PRId64 " ..., terms: ... %" PRId64
-          ", retryWaitMs:%" PRId64 ", mgr: [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64
+          ", retryWaitMs:%" PRId64 ", repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64
          " %" PRId64 ", %" PRId64 ")",
          pNode->vgId, count, pDestId->addr, firstIndex, term, retryWaitMs, pMgr->startIndex, pMgr->matchIndex,
          pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
@ -815,9 +815,9 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
    ASSERT(pMgr->matchIndex == 0);
    if (pMsg->matchIndex < 0) {
      pMgr->restored = true;
-      sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
-            ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
-            pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
+      sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64
+            "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
+            pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
            pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
      return 0;
    }
@ -832,9 +832,9 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
    if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) {
      pMgr->matchIndex = pMsg->matchIndex;
      pMgr->restored = true;
-      sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
-            ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
-            pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
+      sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64
+            "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
+            pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
            pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
      return 0;
    }
@ -958,10 +958,10 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde
  pMgr->endIndex = index + 1;

  SSyncLogBuffer* pBuf = pNode->pLogBuf;
-  sTrace("vgId:%d, probe peer:%" PRIx64 " with msg of index:%" PRId64 " term:%" PRId64 ". mgr (rs:%d): [%" PRId64
+  sTrace("vgId:%d, probe peer:%" PRIx64 " with msg of index:%" PRId64 " term:%" PRId64 ". repl-mgr:[%" PRId64
         " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
-        pNode->vgId, pDestId->addr, index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
-        pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
+         pNode->vgId, pDestId->addr, index, term, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex,
+         pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
  return 0;
 }

@ -1002,9 +1002,9 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {

    pMgr->endIndex = index + 1;
    if (barrier) {
-      sInfo("vgId:%d, replicated sync barrier to dnode:%d. index:%" PRId64 ", term:%" PRId64
-            ", repl mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 ")",
-            pNode->vgId, DID(pDestId), index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex);
+      sInfo("vgId:%d, replicated sync barrier to dnode:%d. index:%" PRId64 ", term:%" PRId64 ", repl-mgr:[%" PRId64
+            " %" PRId64 ", %" PRId64 ")",
+            pNode->vgId, DID(pDestId), index, term, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex);
      break;
    }
  }
@ -1013,10 +1013,10 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {

  SSyncLogBuffer* pBuf = pNode->pLogBuf;
  sTrace("vgId:%d, replicated %d msgs to peer:%" PRIx64 ". indexes:%" PRId64 "..., terms: ...%" PRId64
-         ", mgr: (rs:%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64
+         ", repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64
         ")",
-         pNode->vgId, count, pDestId->addr, firstIndex, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex,
-         pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
+         pNode->vgId, count, pDestId->addr, firstIndex, term, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
+         pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
  return 0;
 }

--- a/source/libs/sync/src/syncSnapshot.c
+++ b/source/libs/sync/src/syncSnapshot.c
@ -23,6 +23,44 @@
 #include "syncReplication.h"
 #include "syncUtil.h"

+static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) {
+  taosThreadMutexLock(&pBuf->mutex);
+  for (int64_t i = pBuf->start; i < pBuf->end; ++i) {
+    if (pBuf->entryDeleteCb) {
+      pBuf->entryDeleteCb(pBuf->entries[i % pBuf->size]);
+    }
+    pBuf->entries[i % pBuf->size] = NULL;
+  }
+  pBuf->start = SYNC_SNAPSHOT_SEQ_BEGIN + 1;
+  pBuf->end = pBuf->start;
+  pBuf->cursor = pBuf->start - 1;
+  taosThreadMutexUnlock(&pBuf->mutex);
+}
+
+static void syncSnapBufferDestroy(SSyncSnapBuffer **ppBuf) {
+  if (ppBuf == NULL || ppBuf[0] == NULL) return;
+  SSyncSnapBuffer *pBuf = ppBuf[0];
+
+  syncSnapBufferReset(pBuf);
+
+  taosThreadMutexDestroy(&pBuf->mutex);
+  taosMemoryFree(ppBuf[0]);
+  ppBuf[0] = NULL;
+  return;
+}
+
+static SSyncSnapBuffer *syncSnapBufferCreate() {
+  SSyncSnapBuffer *pBuf = taosMemoryCalloc(1, sizeof(SSyncSnapBuffer));
+  if (pBuf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  pBuf->size = sizeof(pBuf->entries) / sizeof(void *);
+  ASSERT(pBuf->size == TSDB_SYNC_SNAP_BUFFER_SIZE);
+  taosThreadMutexInit(&pBuf->mutex, NULL);
+  return pBuf;
+}
+
 SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) {
  bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) &&
                   (pSyncNode->pFsm->FpSnapshotDoRead != NULL);
@ -38,8 +76,6 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
  pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID;
  pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
  pSender->pReader = NULL;
-  pSender->pCurrentBlock = NULL;
-  pSender->blockLen = 0;
  pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS;
  pSender->pSyncNode = pSyncNode;
  pSender->replicaIndex = replicaIndex;
@ -49,24 +85,42 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
  pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot);
  pSender->finish = false;

+  SSyncSnapBuffer *pSndBuf = syncSnapBufferCreate();
+  if (pSndBuf == NULL) {
+    taosMemoryFree(pSender);
+    pSender = NULL;
+    return NULL;
+  }
+  pSndBuf->entryDeleteCb = syncSnapBlockDestroy;
+  pSender->pSndBuf = pSndBuf;
+
+  syncSnapBufferReset(pSender->pSndBuf);
  return pSender;
 }

+void syncSnapBlockDestroy(void *ptr) {
+  SyncSnapBlock *pBlk = ptr;
+  if (pBlk->pBlock != NULL) {
+    taosMemoryFree(pBlk->pBlock);
+    pBlk->pBlock = NULL;
+    pBlk->blockLen = 0;
+  }
+  taosMemoryFree(pBlk);
+}
+
 void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
  if (pSender == NULL) return;

-  // free current block
-  if (pSender->pCurrentBlock != NULL) {
-    taosMemoryFree(pSender->pCurrentBlock);
-    pSender->pCurrentBlock = NULL;
-  }
-
  // close reader
  if (pSender->pReader != NULL) {
    pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
    pSender->pReader = NULL;
  }

+  // free snap buffer
+  if (pSender->pSndBuf) {
+    syncSnapBufferDestroy(&pSender->pSndBuf);
+  }
  // free sender
  taosMemoryFree(pSender);
 }
@ -79,11 +133,9 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) {
  int8_t started = atomic_val_compare_exchange_8(&pSender->start, false, true);
  if (started) return 0;

-  pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
+  pSender->seq = SYNC_SNAPSHOT_SEQ_PREP;
  pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
  pSender->pReader = NULL;
-  pSender->pCurrentBlock = NULL;
-  pSender->blockLen = 0;
  pSender->snapshotParam.start = SYNC_INDEX_INVALID;
  pSender->snapshotParam.end = SYNC_INDEX_INVALID;
  pSender->snapshot.data = NULL;
@ -127,29 +179,28 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) {
  SyncSnapshotSend *pMsg = rpcMsg.pCont;
  pMsg->srcId = pSender->pSyncNode->myRaftId;
  pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex];
-  pMsg->term = raftStoreGetTerm(pSender->pSyncNode);
+  pMsg->term = pSender->term;
  pMsg->beginIndex = pSender->snapshotParam.start;
  pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
  pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
  pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
  pMsg->lastConfig = pSender->lastConfig;
  pMsg->startTime = pSender->startTime;
-  pMsg->seq = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT;
+  pMsg->seq = pSender->seq;

  if (dataLen > 0) {
    pMsg->payloadType = snapInfo.type;
    memcpy(pMsg->data, snapInfo.data, dataLen);
  }

-  // event log
-  syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender start");
-
  // send msg
  if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) {
    sSError(pSender, "snapshot sender send msg failed since %s", terrstr());
    goto _out;
  }

+  sSInfo(pSender, "snapshot sender start, to dnode:%d.", DID(&pMsg->destId));
+
  code = 0;
 _out:
  if (snapInfo.data) {
@ -175,48 +226,59 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) {
    pSender->pReader = NULL;
  }

-  // free current block
-  if (pSender->pCurrentBlock != NULL) {
-    taosMemoryFree(pSender->pCurrentBlock);
-    pSender->pCurrentBlock = NULL;
-    pSender->blockLen = 0;
-  }
+  syncSnapBufferReset(pSender->pSndBuf);
+
+  SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex];
+  sSInfo(pSender, "snapshot sender stop, to dnode:%d, finish:%d", DID(&destId), finish);
 }

 // when sender receive ack, call this function to send msg from seq
 // seq = ack + 1, already updated
 static int32_t snapshotSend(SSyncSnapshotSender *pSender) {
-  // free memory last time (current seq - 1)
-  if (pSender->pCurrentBlock != NULL) {
-    taosMemoryFree(pSender->pCurrentBlock);
-    pSender->pCurrentBlock = NULL;
-    pSender->blockLen = 0;
+  int32_t        code = -1;
+  SyncSnapBlock *pBlk = NULL;
+
+  if (pSender->seq < SYNC_SNAPSHOT_SEQ_END) {
+    pSender->seq++;
+
+    if (pSender->seq > SYNC_SNAPSHOT_SEQ_BEGIN) {
+      pBlk = taosMemoryCalloc(1, sizeof(SyncSnapBlock));
+      if (pBlk == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        goto _OUT;
      }

+      pBlk->seq = pSender->seq;
+
      // read data
      int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader,
-                                                           &pSender->pCurrentBlock, &pSender->blockLen);
+                                                               &pBlk->pBlock, &pBlk->blockLen);
      if (ret != 0) {
        sSError(pSender, "snapshot sender read failed since %s", terrstr());
-    return -1;
+        goto _OUT;
      }

-  if (pSender->blockLen > 0) {
+      if (pBlk->blockLen > 0) {
        // has read data
-    sSDebug(pSender, "vgId:%d, snapshot sender continue to read, blockLen:%d seq:%d", pSender->pSyncNode->vgId,
-            pSender->blockLen, pSender->seq);
+        sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pBlk->blockLen, pBlk->seq);
      } else {
        // read finish, update seq to end
        pSender->seq = SYNC_SNAPSHOT_SEQ_END;
-    sSInfo(pSender, "vgId:%d, snapshot sender read to the end, blockLen:%d seq:%d", pSender->pSyncNode->vgId,
-           pSender->blockLen, pSender->seq);
+        sSInfo(pSender, "snapshot sender read to the end");
+        code = 0;
+        goto _OUT;
+      }
+    }
  }

+  ASSERT(pSender->seq >= SYNC_SNAPSHOT_SEQ_BEGIN && pSender->seq <= SYNC_SNAPSHOT_SEQ_END);
+
+  int32_t blockLen = (pBlk != NULL) ? pBlk->blockLen : 0;
  // build msg
  SRpcMsg rpcMsg = {0};
-  if (syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId) != 0) {
+  if (syncBuildSnapshotSend(&rpcMsg, blockLen, pSender->pSyncNode->vgId) != 0) {
    sSError(pSender, "vgId:%d, snapshot sender build msg failed since %s", pSender->pSyncNode->vgId, terrstr());
-    return -1;
+    goto _OUT;
  }

  SyncSnapshotSend *pMsg = rpcMsg.pCont;
@ -231,77 +293,83 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) {
  pMsg->startTime = pSender->startTime;
  pMsg->seq = pSender->seq;

-  if (pSender->pCurrentBlock != NULL) {
-    memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
-  }
-
-  // event log
-  if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) {
-    syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender finish");
-  } else {
-    syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender sending");
+  if (pBlk != NULL && pBlk->pBlock != NULL && pBlk->blockLen > 0) {
+    memcpy(pMsg->data, pBlk->pBlock, pBlk->blockLen);
  }

  // send msg
  if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) {
    sSError(pSender, "snapshot sender send msg failed since %s", terrstr());
-    return -1;
+    goto _OUT;
  }

-  pSender->lastSendTime = taosGetTimestampMs();
-  return 0;
+  // put in buffer
+  int64_t nowMs = taosGetTimestampMs();
+  if (pBlk) {
+    ASSERT(pBlk->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pBlk->seq < SYNC_SNAPSHOT_SEQ_END);
+    pBlk->sendTimeMs = nowMs;
+    pSender->pSndBuf->entries[pSender->seq % pSender->pSndBuf->size] = pBlk;
+    pBlk = NULL;
+    pSender->pSndBuf->end = TMAX(pSender->seq + 1, pSender->pSndBuf->end);
+  }
+  pSender->lastSendTime = nowMs;
+  code = 0;
+
+_OUT:;
+  if (pBlk != NULL) {
+    syncSnapBlockDestroy(pBlk);
+    pBlk = NULL;
+  }
+  return code;
 }

 // send snapshot data from cache
 int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
+  SSyncSnapBuffer *pSndBuf = pSender->pSndBuf;
+  int32_t          code = -1;
+  taosThreadMutexLock(&pSndBuf->mutex);
+
+  for (int32_t seq = pSndBuf->cursor + 1; seq < pSndBuf->end; ++seq) {
+    SyncSnapBlock *pBlk = pSndBuf->entries[seq % pSndBuf->size];
+    ASSERT(pBlk && !pBlk->acked);
+    int64_t nowMs = taosGetTimestampMs();
+    if (nowMs < pBlk->sendTimeMs + SYNC_SNAP_RESEND_MS) {
+      continue;
+    }
    // build msg
    SRpcMsg rpcMsg = {0};
-  if (syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId) != 0) {
+    if (syncBuildSnapshotSend(&rpcMsg, pBlk->blockLen, pSender->pSyncNode->vgId) != 0) {
      sSError(pSender, "snapshot sender build msg failed since %s", terrstr());
-    return -1;
+      goto _out;
    }

    SyncSnapshotSend *pMsg = rpcMsg.pCont;
    pMsg->srcId = pSender->pSyncNode->myRaftId;
    pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex];
-  pMsg->term = raftStoreGetTerm(pSender->pSyncNode);
+    pMsg->term = pSender->term;
    pMsg->beginIndex = pSender->snapshotParam.start;
    pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
    pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
    pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
    pMsg->lastConfig = pSender->lastConfig;
    pMsg->startTime = pSender->startTime;
-  pMsg->seq = pSender->seq;
+    pMsg->seq = pBlk->seq;

-  if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) {
-    memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
+    if (pBlk->pBlock != NULL && pBlk->blockLen > 0) {
+      memcpy(pMsg->data, pBlk->pBlock, pBlk->blockLen);
    }

-  // event log
-  syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender resend");
-
    // send msg
    if (syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) {
      sSError(pSender, "snapshot sender resend msg failed since %s", terrstr());
-    return -1;
+      goto _out;
    }
-
-  pSender->lastSendTime = taosGetTimestampMs();
-  return 0;
-}
-
-static int32_t snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
-  if (pMsg->ack != pSender->seq) {
-    sSError(pSender, "snapshot sender update seq failed, ack:%d seq:%d", pMsg->ack, pSender->seq);
-    terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
-    return -1;
+    pBlk->sendTimeMs = nowMs;
  }
-
-  pSender->ack = pMsg->ack;
-  pSender->seq++;
-
-  sSDebug(pSender, "snapshot sender update seq:%d", pSender->seq);
-  return 0;
+  code = 0;
+_out:;
+  taosThreadMutexUnlock(&pSndBuf->mutex);
+  return code;
 }

 // return 0, start ok
@ -328,8 +396,6 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
    return 0;
  }

-  sSInfo(pSender, "snapshot sender start");
-
  int32_t code = snapshotSenderStart(pSender);
  if (code != 0) {
    sSError(pSender, "snapshot sender start error since %s", terrstr());
@ -362,6 +428,16 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from
  pReceiver->snapshot.lastApplyTerm = 0;
  pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID;

+  SSyncSnapBuffer *pRcvBuf = syncSnapBufferCreate();
+  if (pRcvBuf == NULL) {
+    taosMemoryFree(pReceiver);
+    pReceiver = NULL;
+    return NULL;
+  }
+  pRcvBuf->entryDeleteCb = rpcFreeCont;
+  pReceiver->pRcvBuf = pRcvBuf;
+
+  syncSnapBufferReset(pReceiver->pRcvBuf);
  return pReceiver;
 }

@ -389,6 +465,11 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {
    pReceiver->snapshot.data = NULL;
  }

+  // free snap buf
+  if (pReceiver->pRcvBuf) {
+    syncSnapBufferDestroy(&pReceiver->pRcvBuf);
+  }
+
  // free receiver
  taosMemoryFree(pReceiver);
 }
@ -444,20 +525,19 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p
  int8_t started = atomic_val_compare_exchange_8(&pReceiver->start, false, true);
  if (started) return;

-  pReceiver->ack = SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT;
+  pReceiver->ack = SYNC_SNAPSHOT_SEQ_PREP;
  pReceiver->term = pPreMsg->term;
  pReceiver->fromId = pPreMsg->srcId;
  pReceiver->startTime = pPreMsg->startTime;
  ASSERT(pReceiver->startTime);

-  // event log
-  sRInfo(pReceiver, "snapshot receiver is start");
+  sRInfo(pReceiver, "snapshot receiver start, from dnode:%d.", DID(&pReceiver->fromId));
 }

 // just set start = false
 // FpSnapshotStopWrite should not be called
 void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
-  sRInfo(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter);
+  sRDebug(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter);

  int8_t stopped = !atomic_val_compare_exchange_8(&pReceiver->start, true, false);
  if (stopped) return;
@ -472,6 +552,8 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
  } else {
    sRInfo(pReceiver, "snapshot receiver stop, writer is null");
  }
+
+  syncSnapBufferReset(pReceiver->pRcvBuf);
 }

 // when recv last snapshot block, apply data into snapshot
@ -479,7 +561,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
  int32_t code = 0;
  if (pReceiver->pWriter != NULL) {
    // write data
-    sRInfo(pReceiver, "snapshot receiver write finish, blockLen:%d seq:%d", pMsg->dataLen, pMsg->seq);
+    sRInfo(pReceiver, "snapshot receiver write about to finish, blockLen:%d seq:%d", pMsg->dataLen, pMsg->seq);
    if (pMsg->dataLen > 0) {
      code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, pMsg->data,
                                                           pMsg->dataLen);
@ -489,15 +571,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
      }
    }

-    // reset wal
-    sRInfo(pReceiver, "snapshot receiver log restore");
-    code =
-        pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex);
-    if (code != 0) {
-      sRError(pReceiver, "failed to snapshot receiver log restore since %s", terrstr());
-      return -1;
-    }
-
    // update commit index
    if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) {
      pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex;
@ -509,7 +582,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
    }

    // stop writer, apply data
-    sRInfo(pReceiver, "snapshot receiver apply write");
    code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true,
                                                           &pReceiver->snapshot);
    if (code != 0) {
@ -517,21 +589,29 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
      return -1;
    }
    pReceiver->pWriter = NULL;
+    sRInfo(pReceiver, "snapshot receiver write stopped");

    // update progress
    pReceiver->ack = SYNC_SNAPSHOT_SEQ_END;

+    // get fsmState
    SSnapshot snapshot = {0};
    pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot);
    pReceiver->pSyncNode->fsmState = snapshot.state;

+    // reset wal
+    code =
+        pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex);
+    if (code != 0) {
+      sRError(pReceiver, "failed to snapshot receiver log restore since %s", terrstr());
+      return -1;
+    }
+    sRInfo(pReceiver, "wal log restored from snapshot");
  } else {
    sRError(pReceiver, "snapshot receiver finish error since writer is null");
    return -1;
  }

-  // event log
-  sRInfo(pReceiver, "snapshot receiver got last data and apply snapshot finished");
  return 0;
 }

@ -599,22 +679,22 @@ static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pM
    int32_t order = 0;
    if ((order = snapshotReceiverSignatureCmp(pReceiver, pMsg)) < 0) {
      sRInfo(pReceiver,
-             "received a new snapshot preparation. restart receiver"
-             "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")",
-             pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime);
+             "received a new snapshot preparation. restart receiver."
+             " msg signature:(%" PRId64 ", %" PRId64 ")",
+             pMsg->term, pMsg->startTime);
      goto _START_RECEIVER;
    } else if (order == 0) {
      sRInfo(pReceiver,
-             "received a duplicate snapshot preparation. send reply"
-             "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")",
-             pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime);
+             "received a duplicate snapshot preparation. send reply."
+             " msg signature:(%" PRId64 ", %" PRId64 ")",
+             pMsg->term, pMsg->startTime);
      goto _SEND_REPLY;
    } else {
      // ignore
      sRError(pReceiver,
-              "received a stale snapshot preparation. ignore"
-              "receiver signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")",
-              pReceiver->term, pReceiver->startTime, pMsg->term, pMsg->startTime);
+              "received a stale snapshot preparation. ignore."
+              " msg signature:(%" PRId64 ", %" PRId64 ")",
+              pMsg->term, pMsg->startTime);
      terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
      code = terrno;
      goto _SEND_REPLY;
@ -765,29 +845,8 @@ _SEND_REPLY:
  return code;
 }

-static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
-  // condition 4
-  // transfering
-  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
-  int64_t timeNow = taosGetTimestampMs();
-  int32_t                code = 0;
-
-  if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) {
-    terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
-    sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr());
-    code = terrno;
-    goto _SEND_REPLY;
-  }
-
-  if (snapshotReceiverGotData(pReceiver, pMsg) != 0) {
-    code = terrno;
-    if (code >= SYNC_SNAPSHOT_SEQ_INVALID) {
-      code = TSDB_CODE_SYN_INTERNAL_ERROR;
-    }
-  }
-
-_SEND_REPLY:;
-
+static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, int32_t code) {
+  SSyncNode *pSyncNode = pReceiver->pSyncNode;
  // build msg
  SRpcMsg rpcMsg = {0};
  if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) {
@ -811,10 +870,79 @@ _SEND_REPLY:;
    sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr());
    return -1;
  }
+  return 0;
+}

+static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend **ppMsg) {
+  int32_t           code = 0;
+  SSyncSnapBuffer  *pRcvBuf = pReceiver->pRcvBuf;
+  SyncSnapshotSend *pMsg = ppMsg[0];
+  terrno = TSDB_CODE_SUCCESS;
+
+  taosThreadMutexLock(&pRcvBuf->mutex);
+
+  if (pMsg->seq - pRcvBuf->start >= pRcvBuf->size) {
+    terrno = TSDB_CODE_SYN_BUFFER_FULL;
+    code = terrno;
+    goto _out;
+  }
+
+  ASSERT(pRcvBuf->start <= pRcvBuf->cursor + 1 && pRcvBuf->cursor < pRcvBuf->end);
+
+  if (pMsg->seq > pRcvBuf->cursor) {
+    pRcvBuf->entries[pMsg->seq % pRcvBuf->size] = pMsg;
+    ppMsg[0] = NULL;
+    pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end);
+  } else {
+    syncSnapSendRsp(pReceiver, pMsg, code);
+    goto _out;
+  }
+
+  for (int64_t seq = pRcvBuf->cursor + 1; seq < pRcvBuf->end; ++seq) {
+    if (pRcvBuf->entries[seq]) {
+      pRcvBuf->cursor = seq;
+    } else {
+      break;
+    }
+  }
+
+  for (int64_t seq = pRcvBuf->start; seq <= pRcvBuf->cursor; ++seq) {
+    if (snapshotReceiverGotData(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size]) != 0) {
+      code = terrno;
+      if (code >= SYNC_SNAPSHOT_SEQ_INVALID) {
+        code = TSDB_CODE_SYN_INTERNAL_ERROR;
+      }
+    }
+    pRcvBuf->start = seq + 1;
+    syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code);
+    pRcvBuf->entryDeleteCb(pRcvBuf->entries[seq % pRcvBuf->size]);
+    pRcvBuf->entries[seq % pRcvBuf->size] = NULL;
+    if (code) goto _out;
+  }
+
+_out:
+  taosThreadMutexUnlock(&pRcvBuf->mutex);
  return code;
 }

+static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend **ppMsg) {
+  // condition 4
+  // transfering
+  SyncSnapshotSend *pMsg = ppMsg[0];
+  ASSERT(pMsg);
+  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
+  int64_t                timeNow = taosGetTimestampMs();
+  int32_t                code = 0;
+
+  if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) {
+    terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
+    sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr());
+    return syncSnapSendRsp(pReceiver, pMsg, terrno);
+  }
+
+  return syncSnapBufferRecv(pReceiver, ppMsg);
+}
+
 static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
  // condition 2
  // end, finish FSM
@ -867,7 +995,7 @@ _SEND_REPLY:;

 // receiver on message
 //
-// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT
+// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP
 //              if receiver already start
 //                    if sender.start-time > receiver.start-time, restart receiver(reply snapshot start)
 //                    if sender.start-time = receiver.start-time, maybe duplicate msg
@ -885,9 +1013,11 @@ _SEND_REPLY:;
 //
 // condition 5, got data, update ack
 //
-int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
-  SyncSnapshotSend      *pMsg = pRpcMsg->pCont;
+int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
+  SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont;
+  SyncSnapshotSend      *pMsg = ppMsg[0];
  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
+  int32_t                code = 0;

  // if already drop replica, do not process
  if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) {
@ -911,49 +1041,56 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
    syncNodeUpdateTermWithoutStepDown(pSyncNode, pMsg->term);
  }

-  // state, term, seq/ack
-  int32_t code = 0;
-  if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) {
-    if (pMsg->term == raftStoreGetTerm(pSyncNode)) {
-      if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) {
-        sInfo("vgId:%d, receive pre-snapshot msg of snapshot replication. signature:(%" PRId64 ", %" PRId64 ")",
-              pSyncNode->vgId, pMsg->term, pMsg->startTime);
-        code = syncNodeOnSnapshotPrep(pSyncNode, pMsg);
-      } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) {
-        sInfo("vgId:%d, receive begin msg of snapshot replication. signature:(%" PRId64 ", %" PRId64 ")",
-              pSyncNode->vgId, pMsg->term, pMsg->startTime);
-        code = syncNodeOnSnapshotBegin(pSyncNode, pMsg);
-      } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
-        sInfo("vgId:%d, receive end msg of snapshot replication. signature: (%" PRId64 ", %" PRId64 ")",
-              pSyncNode->vgId, pMsg->term, pMsg->startTime);
-        code = syncNodeOnSnapshotEnd(pSyncNode, pMsg);
-        if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) {
-          sRError(pReceiver, "failed to reinit log buffer since %s", terrstr());
-          code = -1;
-        }
-      } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) {
-        // force close, no response
-        syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process force stop");
-        snapshotReceiverStop(pReceiver);
-      } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) {
-        syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq data");
-        code = syncNodeOnSnapshotReceive(pSyncNode, pMsg);
-      } else {
-        // error log
-        sRError(pReceiver, "snapshot receiver recv error seq:%d, my ack:%d", pMsg->seq, pReceiver->ack);
-        code = -1;
-      }
-    } else {
-      // error log
-      sRError(pReceiver, "snapshot receiver term not equal");
-      code = -1;
-    }
-  } else {
-    // error log
-    sRError(pReceiver, "snapshot receiver not follower");
-    code = -1;
+  if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER && pSyncNode->state != TAOS_SYNC_STATE_LEARNER) {
+    sRError(pReceiver, "snapshot receiver not a follower or learner");
+    return -1;
  }

+  if (pMsg->seq < SYNC_SNAPSHOT_SEQ_PREP || pMsg->seq > SYNC_SNAPSHOT_SEQ_END) {
+    sRError(pReceiver, "snap replication msg with invalid seq:%d", pMsg->seq);
+    return -1;
+  }
+
+  // prepare
+  if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP) {
+    sInfo("vgId:%d, prepare snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term,
+          pMsg->startTime);
+    code = syncNodeOnSnapshotPrep(pSyncNode, pMsg);
+    goto _out;
+  }
+
+  // begin
+  if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) {
+    sInfo("vgId:%d, begin snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term,
+          pMsg->startTime);
+    code = syncNodeOnSnapshotBegin(pSyncNode, pMsg);
+    goto _out;
+  }
+
+  // data
+  if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) {
+    code = syncNodeOnSnapshotReceive(pSyncNode, ppMsg);
+    goto _out;
+  }
+
+  // end
+  if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
+    sInfo("vgId:%d, end snap replication. msg signature:(%" PRId64 ", %" PRId64 ")", pSyncNode->vgId, pMsg->term,
+          pMsg->startTime);
+    code = syncNodeOnSnapshotEnd(pSyncNode, pMsg);
+    if (code != 0) {
+      sRError(pReceiver, "failed to end snapshot.");
+      goto _out;
+    }
+
+    code = syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode);
+    if (code != 0) {
+      sRError(pReceiver, "failed to reinit log buffer since %s", terrstr());
+    }
+    goto _out;
+  }
+
+_out:;
  syncNodeResetElectTimer(pSyncNode);
  return code;
 }
@ -993,41 +1130,7 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend
  // update next index
  syncIndexMgrSetIndex(pSyncNode->pNextIndex, &pMsg->srcId, snapshot.lastApplyIndex + 1);

-  // update seq
-  pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
-
-  // build begin msg
-  SRpcMsg rpcMsg = {0};
-  if (syncBuildSnapshotSend(&rpcMsg, 0, pSender->pSyncNode->vgId) != 0) {
-    sSError(pSender, "prepare snapshot failed since build msg error");
-    return -1;
-  }
-
-  SyncSnapshotSend *pSendMsg = rpcMsg.pCont;
-  pSendMsg->srcId = pSender->pSyncNode->myRaftId;
-  pSendMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex];
-  pSendMsg->term = raftStoreGetTerm(pSender->pSyncNode);
-  pSendMsg->beginIndex = pSender->snapshotParam.start;
-  pSendMsg->lastIndex = pSender->snapshot.lastApplyIndex;
-  pSendMsg->lastTerm = pSender->snapshot.lastApplyTerm;
-  pSendMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
-  pSendMsg->lastConfig = pSender->lastConfig;
-  pSendMsg->startTime = pSender->startTime;
-  pSendMsg->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
-
-  ASSERT(pSendMsg->startTime);
-
-  sSInfo(pSender, "begin snapshot replication to dnode %d. startTime:%" PRId64, DID(&pSendMsg->destId),
-         pSendMsg->startTime);
-
-  // send msg
-  syncLogSendSyncSnapshotSend(pSyncNode, pSendMsg, "snapshot sender reply pre");
-  if (syncNodeSendMsgById(&pSendMsg->destId, pSender->pSyncNode, &rpcMsg) != 0) {
-    sSError(pSender, "prepare snapshot failed since send msg error");
-    return -1;
-  }
-
-  return 0;
+  return snapshotSend(pSender);
 }

 static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
@ -1038,14 +1141,77 @@ static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnap
  return 0;
 }

+static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp **ppMsg) {
+  int32_t          code = 0;
+  SSyncSnapBuffer *pSndBuf = pSender->pSndBuf;
+  SyncSnapshotRsp *pMsg = ppMsg[0];
+
+  taosThreadMutexLock(&pSndBuf->mutex);
+  if (snapshotSenderSignatureCmp(pSender, pMsg) != 0) {
+    code = terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
+    goto _out;
+  }
+
+  if (pSender->pReader == NULL || pSender->finish) {
+    code = terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
+    goto _out;
+  }
+
+  if (pMsg->ack - pSndBuf->start >= pSndBuf->size) {
+    code = terrno = TSDB_CODE_SYN_BUFFER_FULL;
+    goto _out;
+  }
+
+  ASSERT(pSndBuf->start <= pSndBuf->cursor + 1 && pSndBuf->cursor < pSndBuf->end);
+
+  if (pMsg->ack > pSndBuf->cursor && pMsg->ack < pSndBuf->end) {
+    SyncSnapBlock *pBlk = pSndBuf->entries[pMsg->ack % pSndBuf->size];
+    ASSERT(pBlk);
+    pBlk->acked = 1;
+  }
+
+  for (int64_t ack = pSndBuf->cursor + 1; ack < pSndBuf->end; ++ack) {
+    SyncSnapBlock *pBlk = pSndBuf->entries[ack % pSndBuf->size];
+    if (pBlk->acked) {
+      pSndBuf->cursor = ack;
+    } else {
+      break;
+    }
+  }
+
+  for (int64_t ack = pSndBuf->start; ack <= pSndBuf->cursor; ++ack) {
+    pSndBuf->entryDeleteCb(pSndBuf->entries[ack % pSndBuf->size]);
+    pSndBuf->entries[ack % pSndBuf->size] = NULL;
+    pSndBuf->start = ack + 1;
+  }
+
+  while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < (pSndBuf->size >> 2)) {
+    if (snapshotSend(pSender) != 0) {
+      code = terrno;
+      goto _out;
+    }
+  }
+
+  if (pSender->seq == SYNC_SNAPSHOT_SEQ_END && pSndBuf->end <= pSndBuf->start) {
+    if (snapshotSend(pSender) != 0) {
+      code = terrno;
+      goto _out;
+    }
+  }
+_out:
+  taosThreadMutexUnlock(&pSndBuf->mutex);
+  return code;
+}
+
 // sender on message
 //
 // condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender
 // condition 2 sender receives ack, set seq = ack + 1, send msg from seq
 // condition 3 sender receives error msg, just print error log
 //
-int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
-  SyncSnapshotRsp *pMsg = pRpcMsg->pCont;
+int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
+  SyncSnapshotRsp **ppMsg = (SyncSnapshotRsp **)&pRpcMsg->pCont;
+  SyncSnapshotRsp  *pMsg = ppMsg[0];

  // if already drop replica, do not process
  if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) {
@ -1071,10 +1237,7 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
  // check signature
  int32_t order = 0;
  if ((order = snapshotSenderSignatureCmp(pSender, pMsg)) > 0) {
-    sSError(pSender,
-            "received a stale snapshot rsp. ignore it"
-            "sender signature: (%" PRId64 ", %" PRId64 "), msg signature:(%" PRId64 ", %" PRId64 ")",
-            pSender->term, pSender->startTime, pMsg->term, pMsg->startTime);
+    sSWarn(pSender, "ignore a stale snap rsp, msg signature:(%" PRId64 ", %" PRId64 ").", pMsg->term, pMsg->startTime);
    terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
    return -1;
  } else if (order < 0) {
@ -1083,9 +1246,7 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
    goto _ERROR;
  }

-  // state, term, seq/ack
  if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender not leader");
    sSError(pSender, "snapshot sender not leader");
    terrno = TSDB_CODE_SYN_NOT_LEADER;
    goto _ERROR;
@ -1093,83 +1254,46 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {

  SyncTerm currentTerm = raftStoreGetTerm(pSyncNode);
  if (pMsg->term != currentTerm) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver term not match");
-    sSError(pSender, "snapshot sender term not equal, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term,
+    sSError(pSender, "snapshot sender term mismatch, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term,
            currentTerm);
    terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
    goto _ERROR;
  }

  if (pMsg->code != 0) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error code");
    sSError(pSender, "snapshot sender receive error:%s 0x%x and stop sender", tstrerror(pMsg->code), pMsg->code);
    terrno = pMsg->code;
    goto _ERROR;
  }

-  // prepare <begin, end>, send begin msg
-  if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq pre-snapshot");
-    return syncNodeOnSnapshotPrepRsp(pSyncNode, pSender, pMsg);
-  }
-
-  if (pSender->pReader == NULL || pSender->finish) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender invalid");
-    sSError(pSender, "snapshot sender invalid error:%s 0x%x, pReader:%p finish:%d", tstrerror(pMsg->code), pMsg->code,
-            pSender->pReader, pSender->finish);
-    terrno = pMsg->code;
+  // send begin
+  if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PREP) {
+    sSInfo(pSender, "process prepare rsp");
+    if (syncNodeOnSnapshotPrepRsp(pSyncNode, pSender, pMsg) != 0) {
      goto _ERROR;
    }
-
-  if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin");
-    if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) {
-      return -1;
  }

-    if (snapshotSend(pSender) != 0) {
-      return -1;
+  // send msg of data or end
+  if (pMsg->ack >= SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->ack < SYNC_SNAPSHOT_SEQ_END) {
+    if (syncSnapBufferSend(pSender, ppMsg) != 0) {
+      sSError(pSender, "failed to replicate snap since %s. seq:%d, pReader:%p, finish:%d", terrstr(), pSender->seq,
+              pSender->pReader, pSender->finish);
+      goto _ERROR;
    }
-    return 0;
  }

-  // receive ack is finish, close sender
+  // end
  if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq end");
+    sSInfo(pSender, "process end rsp");
    snapshotSenderStop(pSender, true);
    syncNodeReplicateReset(pSyncNode, &pMsg->srcId);
-    return 0;
-  }
-
-  // send next msg
-  if (pMsg->ack == pSender->seq) {
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq data");
-    // update sender ack
-    if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) {
-      return -1;
-    }
-    if (snapshotSend(pSender) != 0) {
-      return -1;
-    }
-  } else if (pMsg->ack == pSender->seq - 1) {
-    // maybe resend
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq and resend");
-    if (snapshotReSend(pSender) != 0) {
-      return -1;
-    }
-  } else {
-    // error log
-    syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error ack");
-    sSError(pSender, "snapshot sender receive error ack:%d, my seq:%d", pMsg->ack, pSender->seq);
-    snapshotSenderStop(pSender, true);
-    syncNodeReplicateReset(pSyncNode, &pMsg->srcId);
-    return -1;
  }

  return 0;

 _ERROR:
-  snapshotSenderStop(pSender, true);
+  snapshotSenderStop(pSender, false);
  syncNodeReplicateReset(pSyncNode, &pMsg->srcId);
  return -1;
 }
--- a/source/libs/sync/src/syncTimeout.c
+++ b/source/libs/sync/src/syncTimeout.c
@ -77,12 +77,19 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) {
  for (int i = 0; i < ths->peersNum; ++i) {
    SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(ths->peersId[i]));
    if (pSender != NULL) {
-      if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start &&
-          timeNow - pSender->lastSendTime > SYNC_SNAP_RESEND_MS) {
-        snapshotReSend(pSender);
+      if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start) {
+        int64_t elapsedMs = timeNow - pSender->lastSendTime;
+        if (elapsedMs < SYNC_SNAP_RESEND_MS) {
+          continue;
+        }
+
+        if (elapsedMs > SYNC_SNAP_TIMEOUT_MS) {
+          sSError(pSender, "snap replication timeout, terminate.");
+          snapshotSenderStop(pSender, false);
        } else {
-        sTrace("vgId:%d, do not resend: nstart%d, now:%" PRId64 ", lstsend:%" PRId64 ", diff:%" PRId64, ths->vgId,
-               ths->isStart, timeNow, pSender->lastSendTime, timeNow - pSender->lastSendTime);
+          sSWarn(pSender, "snap replication resend.");
+          snapshotReSend(pSender);
+        }
      }
    }
  }
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@ -267,21 +267,23 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla
  va_end(argpointer);

  taosPrintLog(flags, level, dflag,
-               "vgId:%d, %s, sync:%s, snap-sender:{%p start:%" PRId64 " end:%" PRId64 " last-index:%" PRId64
-               " last-term:%" PRIu64 " last-cfg:%" PRId64
-               ", seq:%d ack:%d finish:%d, as:%d dnode:%d}"
+               "vgId:%d, %s, sync:%s, snap-sender:%p signature:(%" PRId64 ", %" PRId64 "), {start:%" PRId64
+               " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64
+               ", seq:%d, ack:%d, "
+               " buf:[%" PRId64 " %" PRId64 ", %" PRId64
+               "), finish:%d, as:%d, to-dnode:%d}"
               ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64
               ", min-match:%" PRId64 ", snap:{last-index:%" PRId64 ", term:%" PRIu64
               "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64
-               ", chging:%d, restore:%d, quorum:%d, lc-timer:{elect:%" PRId64 ", hb:%" PRId64 "}, peer:%s, cfg:%s",
-               pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->snapshotParam.start,
-               pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm,
-               pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->replicaIndex,
-               DID(&pNode->replicasId[pSender->replicaIndex]), raftStoreGetTerm(pNode), pNode->commitIndex,
-               logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm,
-               pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, pNode->replicaNum, pNode->raftCfg.lastConfigIndex,
-               pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock,
-               pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr);
+               ", chging:%d, restore:%d, quorum:%d, peer:%s, cfg:%s",
+               pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->term, pSender->startTime,
+               pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex,
+               pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack,
+               pSender->pSndBuf->start, pSender->pSndBuf->cursor, pSender->pSndBuf->end, pSender->finish,
+               pSender->replicaIndex, DID(&pNode->replicasId[pSender->replicaIndex]), raftStoreGetTerm(pNode),
+               pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex,
+               snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize, pNode->replicaNum,
+               pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, pNode->quorum, peerStr, cfgStr);
 }

 void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotReceiver* pReceiver,
@ -316,19 +318,21 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df
  taosPrintLog(
      flags, level, dflag,
      "vgId:%d, %s, sync:%s,"
-      " snap-receiver:{%p started:%d acked:%d term:%" PRIu64 " start-time:%" PRId64 " from-dnode:%d, start:%" PRId64
-      " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64
+      " snap-receiver:%p signature:(%" PRId64 ", %" PRId64 "), {start:%d ack:%d buf:[%" PRId64 " %" PRId64 ", %" PRId64
+      ")"
+      " from-dnode:%d, start:%" PRId64 " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64
      "}"
      ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 ", min-match:%" PRId64
      ", snap:{last-index:%" PRId64 ", last-term:%" PRIu64 "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64
-      ", chging:%d, restore:%d, quorum:%d, lc-timers:{elect:%" PRId64 ", hb:%" PRId64 "}, peer:%s, cfg:%s",
-      pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term,
-      pReceiver->startTime, DID(&pReceiver->fromId), pReceiver->snapshotParam.start, pReceiver->snapshotParam.end,
+      ", chging:%d, restore:%d, quorum:%d, peer:%s, cfg:%s",
+      pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->term, pReceiver->startTime, pReceiver->start,
+      pReceiver->ack, pReceiver->pRcvBuf->start, pReceiver->pRcvBuf->cursor, pReceiver->pRcvBuf->end,
+      DID(&pReceiver->fromId), pReceiver->snapshotParam.start, pReceiver->snapshotParam.end,
      pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex,
      raftStoreGetTerm(pNode), pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex,
      snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.batchSize,
-      pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish,
-      syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr);
+      pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, pNode->quorum, peerStr,
+      cfgStr);
 }

 void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* s) {
--- a/source/libs/tfs/src/tfsTier.c
+++ b/source/libs/tfs/src/tfsTier.c
@ -110,7 +110,9 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) {
  }

  int32_t retId = -1;
+  int64_t avail = 0;
  for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; ++id) {
+#if 0  // round-robin
    int32_t   diskId = (pTier->nextid + id) % pTier->ndisk;
    STfsDisk *pDisk = pTier->disks[diskId];

@ -126,6 +128,18 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) {
    terrno = 0;
    pTier->nextid = (diskId + 1) % pTier->ndisk;
    break;
+#else  // select the disk with the most available space
+    STfsDisk *pDisk = pTier->disks[id];
+    if (pDisk == NULL) continue;
+
+    if (pDisk->size.avail < tsMinDiskFreeSize) continue;
+
+    if (pDisk->size.avail > avail) {
+      avail = pDisk->size.avail;
+      retId = id;
+      terrno = 0;
+    }
+#endif
  }

  tfsUnLockTier(pTier);
--- a/source/libs/tfs/test/CMakeLists.txt
+++ b/source/libs/tfs/test/CMakeLists.txt
@ -8,7 +8,7 @@ target_link_libraries(
  PUBLIC gtest_main
 )

-add_test(
-    NAME tfs_test
-    COMMAND tfs_test
-)
+# add_test(
+#     NAME tfs_test
+#     COMMAND tfs_test
+# )
--- a/source/os/src/osTime.c
+++ b/source/os/src/osTime.c
@ -477,7 +477,6 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) {
    return res;
  }
 #ifdef WINDOWS
-  if (*timep < 0) {
  if (*timep < -2208988800LL) {
    if (buf != NULL) {
      sprintf(buf, "NaN");
@ -490,7 +489,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) {
  LARGE_INTEGER offset;
  struct tm     tm1;
  time_t        tt = 0;
-    if (localtime_s(&tm1, &tt) != 0 ) {
+  if (localtime_s(&tm1, &tt) != 0) {
    if (buf != NULL) {
      sprintf(buf, "NaN");
    }
@ -510,14 +509,6 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) {
  result->tm_wday = s.wDayOfWeek;
  result->tm_yday = 0;
  result->tm_isdst = 0;
-  } else {
-    if (localtime_s(result, timep) != 0) {
-      if (buf != NULL) {
-        sprintf(buf, "NaN");
-      }
-      return NULL;
-    }
-  }
 #else
  res = localtime_r(timep, result);
  if (res == NULL && buf != NULL) {
--- a/source/os/test/osTimeTests.cpp
+++ b/source/os/test/osTimeTests.cpp
@ -114,10 +114,6 @@ TEST(osTimeTests, taosLocalTime) {
  ASSERT_EQ(local_time->tm_min, 0);
  ASSERT_EQ(local_time->tm_sec, 0);

-  time_t over_timep = 6406301441633558;
-  local_time = taosLocalTime(&over_timep, &result, NULL);
-  ASSERT_EQ(local_time, nullptr);
-
  time_t neg_timep3 = -78115158887;
  local_time = taosLocalTime(&neg_timep3, &result, NULL);
  ASSERT_EQ(local_time, nullptr);
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@ -191,7 +191,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) {
  }

  if (index >= pArray->size) {
-    uError("index is out of range, current:%" PRIzu " max:%d", index, pArray->capacity);
+    uError("index is out of range, current:%" PRIzu " max:%"PRIzu, index, pArray->size);
    return NULL;
  }

--- a/Show More
+++ b/Show More