fix:conflict from 3.0

2022-07-11 19:06:36 +08:00 · 2022-07-11 19:06:36 +08:00 · f9d5c809c9
parent 4862f032d6 bf09787245
commit f9d5c809c9
201 changed files with 9126 additions and 6017 deletions
--- a/docs/en/12-taos-sql/02-database.md
+++ b/docs/en/12-taos-sql/02-database.md
@ -32,7 +32,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
   - cacheLast: [Description](/reference/config/#cachelast)
   - replica: [Description](/reference/config/#replica)
   - quorum: [Description](/reference/config/#quorum)
-   - maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb)
   - comp: [Description](/reference/config/#comp)
   - precision: [Description](/reference/config/#precision)
 6. Please note that all of the parameters mentioned in this section are configured in configuration file `taos.cfg` on the TDengine server. If not specified in the `create database` statement, the values from taos.cfg are used by default. To override default parameters, they must be specified in the `create database` statement.
--- a/docs/zh/12-taos-sql/02-database.md
+++ b/docs/zh/12-taos-sql/02-database.md
@ -32,7 +32,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
   - cacheLast: [详细说明](/reference/config/#cachelast)
   - replica: [详细说明](/reference/config/#replica)
   - quorum: [详细说明](/reference/config/#quorum)
-   - maxVgroupsPerDb: [详细说明](/reference/config/#maxvgroupsperdb)
   - comp: [详细说明](/reference/config/#comp)
   - precision: [详细说明](/reference/config/#precision)
 6. 请注意上面列出的所有参数都可以配置在配置文件 `taosd.cfg` 中作为创建数据库时使用的默认配置， `create database` 的参数中明确指定的会覆盖配置文件中的设置。
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@ -55,7 +55,7 @@ enum {
 enum {
  STREAM_INPUT__DATA_SUBMIT = 1,
  STREAM_INPUT__DATA_BLOCK,
-  STREAM_INPUT__TABLE_SCAN,
+  // STREAM_INPUT__TABLE_SCAN,
  STREAM_INPUT__TQ_SCAN,
  STREAM_INPUT__DATA_RETRIEVE,
  STREAM_INPUT__TRIGGER,
@ -155,8 +155,8 @@ typedef struct SQueryTableDataCond {
  int32_t      numOfCols;
  SColumnInfo* colList;
  int32_t      type;  // data block load type:
-  int32_t      numOfTWindows;
-  STimeWindow* twindows;
+//  int32_t      numOfTWindows;
+  STimeWindow  twindows;
  int64_t      startVersion;
  int64_t      endVersion;
 } SQueryTableDataCond;
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@ -67,7 +67,6 @@ extern int32_t tsNumOfVnodeQueryThreads;
 extern int32_t tsNumOfVnodeFetchThreads;
 extern int32_t tsNumOfVnodeWriteThreads;
 extern int32_t tsNumOfVnodeSyncThreads;
-extern int32_t tsNumOfVnodeMergeThreads;
 extern int32_t tsNumOfQnodeQueryThreads;
 extern int32_t tsNumOfQnodeFetchThreads;
 extern int32_t tsNumOfSnodeSharedThreads;
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@ -55,11 +55,11 @@ extern int32_t tMsgDict[];

 #define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
 #define TMSG_SEG_SEQ(TYPE)  ((TYPE)&0xff)
-#define TMSG_INFO(TYPE)                                                                                             \
-  ((TYPE) >= 0 &&                                                                                                   \
-   ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \
-    (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG))                         \
-      ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)]                                                \
+#define TMSG_INFO(TYPE)                                                                                      \
+  ((TYPE) >= 0 && ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG ||    \
+                   (TYPE) < TDMT_SCH_MAX_MSG || (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || \
+                   (TYPE) < TDMT_SYNC_MAX_MSG))                                                              \
+      ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)]                                         \
      : 0
 #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))

@ -723,7 +723,7 @@ typedef struct {
  int32_t buffer;        // MB
  int32_t pageSize;
  int32_t pages;
-  int32_t lastRowMem;
+  int32_t cacheLastSize;
  int32_t daysPerFile;
  int32_t daysToKeep0;
  int32_t daysToKeep1;
@ -736,7 +736,7 @@ typedef struct {
  int8_t  compression;
  int8_t  replications;
  int8_t  strict;
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int8_t  schemaless;
  int8_t  ignoreExist;
  int32_t numOfRetensions;
@ -752,7 +752,7 @@ typedef struct {
  int32_t buffer;
  int32_t pageSize;
  int32_t pages;
-  int32_t lastRowMem;
+  int32_t cacheLastSize;
  int32_t daysPerFile;
  int32_t daysToKeep0;
  int32_t daysToKeep1;
@ -760,7 +760,7 @@ typedef struct {
  int32_t fsyncPeriod;
  int8_t  walLevel;
  int8_t  strict;
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int8_t  replications;
 } SAlterDbReq;

@ -815,6 +815,13 @@ typedef struct {
 int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
 int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);

+typedef struct {
+  char db[TSDB_DB_FNAME_LEN];
+} STrimDbReq;
+
+int32_t tSerializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq);
+int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq);
+
 typedef struct {
  int32_t numOfVgroups;
  int32_t numOfStables;
@ -833,7 +840,7 @@ typedef struct {
  int8_t  compression;
  int8_t  replications;
  int8_t  strict;
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int32_t numOfRetensions;
  SArray* pRetensions;
  int8_t  schemaless;
@ -1098,7 +1105,7 @@ typedef struct {
  int32_t  buffer;
  int32_t  pageSize;
  int32_t  pages;
-  int32_t  lastRowMem;
+  int32_t  cacheLastSize;
  int32_t  daysPerFile;
  int32_t  daysToKeep0;
  int32_t  daysToKeep1;
@ -1113,7 +1120,7 @@ typedef struct {
  int8_t   precision;
  int8_t   compression;
  int8_t   strict;
-  int8_t   cacheLastRow;
+  int8_t   cacheLast;
  int8_t   isTsma;
  int8_t   standby;
  int8_t   replica;
@ -1151,7 +1158,7 @@ typedef struct {
  int32_t  buffer;
  int32_t  pageSize;
  int32_t  pages;
-  int32_t  lastRowMem;
+  int32_t  cacheLastSize;
  int32_t  daysPerFile;
  int32_t  daysToKeep0;
  int32_t  daysToKeep1;
@ -1159,7 +1166,7 @@ typedef struct {
  int32_t  fsyncPeriod;
  int8_t   walLevel;
  int8_t   strict;
-  int8_t   cacheLastRow;
+  int8_t   cacheLast;
  int8_t   selfIndex;
  int8_t   replica;
  SReplica replicas[TSDB_MAX_REPLICA];
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@ -116,6 +116,7 @@ enum {
  TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "alter-db", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "sync-db", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "compact-db", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB, "trim-db", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "get-db-cfg", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "vgroup-list", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "create-func", NULL, NULL)
--- a/include/common/ttokendef.h
+++ b/include/common/ttokendef.h
@ -74,201 +74,202 @@
 #define TK_DATABASE       56
 #define TK_USE            57
 #define TK_FLUSH          58
-#define TK_IF             59
-#define TK_NOT            60
-#define TK_EXISTS         61
-#define TK_BUFFER         62
-#define TK_CACHELAST      63
-#define TK_CACHELASTSIZE  64
-#define TK_COMP           65
-#define TK_DURATION       66
-#define TK_NK_VARIABLE    67
-#define TK_FSYNC          68
-#define TK_MAXROWS        69
-#define TK_MINROWS        70
-#define TK_KEEP           71
-#define TK_PAGES          72
-#define TK_PAGESIZE       73
-#define TK_PRECISION      74
-#define TK_REPLICA        75
-#define TK_STRICT         76
-#define TK_WAL            77
-#define TK_VGROUPS        78
-#define TK_SINGLE_STABLE  79
-#define TK_RETENTIONS     80
-#define TK_SCHEMALESS     81
-#define TK_NK_COLON       82
-#define TK_TABLE          83
-#define TK_NK_LP          84
-#define TK_NK_RP          85
-#define TK_STABLE         86
-#define TK_ADD            87
-#define TK_COLUMN         88
-#define TK_MODIFY         89
-#define TK_RENAME         90
-#define TK_TAG            91
-#define TK_SET            92
-#define TK_NK_EQ          93
-#define TK_USING          94
-#define TK_TAGS           95
-#define TK_COMMENT        96
-#define TK_BOOL           97
-#define TK_TINYINT        98
-#define TK_SMALLINT       99
-#define TK_INT            100
-#define TK_INTEGER        101
-#define TK_BIGINT         102
-#define TK_FLOAT          103
-#define TK_DOUBLE         104
-#define TK_BINARY         105
-#define TK_TIMESTAMP      106
-#define TK_NCHAR          107
-#define TK_UNSIGNED       108
-#define TK_JSON           109
-#define TK_VARCHAR        110
-#define TK_MEDIUMBLOB     111
-#define TK_BLOB           112
-#define TK_VARBINARY      113
-#define TK_DECIMAL        114
-#define TK_MAX_DELAY      115
-#define TK_WATERMARK      116
-#define TK_ROLLUP         117
-#define TK_TTL            118
-#define TK_SMA            119
-#define TK_FIRST          120
-#define TK_LAST           121
-#define TK_SHOW           122
-#define TK_DATABASES      123
-#define TK_TABLES         124
-#define TK_STABLES        125
-#define TK_MNODES         126
-#define TK_MODULES        127
-#define TK_QNODES         128
-#define TK_FUNCTIONS      129
-#define TK_INDEXES        130
-#define TK_ACCOUNTS       131
-#define TK_APPS           132
-#define TK_CONNECTIONS    133
-#define TK_LICENCE        134
-#define TK_GRANTS         135
-#define TK_QUERIES        136
-#define TK_SCORES         137
-#define TK_TOPICS         138
-#define TK_VARIABLES      139
-#define TK_BNODES         140
-#define TK_SNODES         141
-#define TK_CLUSTER        142
-#define TK_TRANSACTIONS   143
-#define TK_DISTRIBUTED    144
-#define TK_CONSUMERS      145
-#define TK_SUBSCRIPTIONS  146
-#define TK_LIKE           147
-#define TK_INDEX          148
-#define TK_FUNCTION       149
-#define TK_INTERVAL       150
-#define TK_TOPIC          151
-#define TK_AS             152
-#define TK_WITH           153
-#define TK_META           154
-#define TK_CONSUMER       155
-#define TK_GROUP          156
-#define TK_DESC           157
-#define TK_DESCRIBE       158
-#define TK_RESET          159
-#define TK_QUERY          160
-#define TK_CACHE          161
-#define TK_EXPLAIN        162
-#define TK_ANALYZE        163
-#define TK_VERBOSE        164
-#define TK_NK_BOOL        165
-#define TK_RATIO          166
-#define TK_NK_FLOAT       167
-#define TK_COMPACT        168
-#define TK_VNODES         169
-#define TK_IN             170
-#define TK_OUTPUTTYPE     171
-#define TK_AGGREGATE      172
-#define TK_BUFSIZE        173
-#define TK_STREAM         174
-#define TK_INTO           175
-#define TK_TRIGGER        176
-#define TK_AT_ONCE        177
-#define TK_WINDOW_CLOSE   178
-#define TK_IGNORE         179
-#define TK_EXPIRED        180
-#define TK_KILL           181
-#define TK_CONNECTION     182
-#define TK_TRANSACTION    183
-#define TK_BALANCE        184
-#define TK_VGROUP         185
-#define TK_MERGE          186
-#define TK_REDISTRIBUTE   187
-#define TK_SPLIT          188
-#define TK_SYNCDB         189
-#define TK_DELETE         190
-#define TK_INSERT         191
-#define TK_NULL           192
-#define TK_NK_QUESTION    193
-#define TK_NK_ARROW       194
-#define TK_ROWTS          195
-#define TK_TBNAME         196
-#define TK_QSTARTTS       197
-#define TK_QENDTS         198
-#define TK_WSTARTTS       199
-#define TK_WENDTS         200
-#define TK_WDURATION      201
-#define TK_CAST           202
-#define TK_NOW            203
-#define TK_TODAY          204
-#define TK_TIMEZONE       205
-#define TK_CLIENT_VERSION 206
-#define TK_SERVER_VERSION 207
-#define TK_SERVER_STATUS  208
-#define TK_CURRENT_USER   209
-#define TK_COUNT          210
-#define TK_LAST_ROW       211
-#define TK_BETWEEN        212
-#define TK_IS             213
-#define TK_NK_LT          214
-#define TK_NK_GT          215
-#define TK_NK_LE          216
-#define TK_NK_GE          217
-#define TK_NK_NE          218
-#define TK_MATCH          219
-#define TK_NMATCH         220
-#define TK_CONTAINS       221
-#define TK_JOIN           222
-#define TK_INNER          223
-#define TK_SELECT         224
-#define TK_DISTINCT       225
-#define TK_WHERE          226
-#define TK_PARTITION      227
-#define TK_BY             228
-#define TK_SESSION        229
-#define TK_STATE_WINDOW   230
-#define TK_SLIDING        231
-#define TK_FILL           232
-#define TK_VALUE          233
-#define TK_NONE           234
-#define TK_PREV           235
-#define TK_LINEAR         236
-#define TK_NEXT           237
-#define TK_HAVING         238
-#define TK_RANGE          239
-#define TK_EVERY          240
-#define TK_ORDER          241
-#define TK_SLIMIT         242
-#define TK_SOFFSET        243
-#define TK_LIMIT          244
-#define TK_OFFSET         245
-#define TK_ASC            246
-#define TK_NULLS          247
-#define TK_ID             248
-#define TK_NK_BITNOT      249
-#define TK_VALUES         250
-#define TK_IMPORT         251
-#define TK_NK_SEMI        252
-#define TK_FILE           253
+#define TK_TRIM           59
+#define TK_IF             60
+#define TK_NOT            61
+#define TK_EXISTS         62
+#define TK_BUFFER         63
+#define TK_CACHELAST      64
+#define TK_CACHELASTSIZE  65
+#define TK_COMP           66
+#define TK_DURATION       67
+#define TK_NK_VARIABLE    68
+#define TK_FSYNC          69
+#define TK_MAXROWS        70
+#define TK_MINROWS        71
+#define TK_KEEP           72
+#define TK_PAGES          73
+#define TK_PAGESIZE       74
+#define TK_PRECISION      75
+#define TK_REPLICA        76
+#define TK_STRICT         77
+#define TK_WAL            78
+#define TK_VGROUPS        79
+#define TK_SINGLE_STABLE  80
+#define TK_RETENTIONS     81
+#define TK_SCHEMALESS     82
+#define TK_NK_COLON       83
+#define TK_TABLE          84
+#define TK_NK_LP          85
+#define TK_NK_RP          86
+#define TK_STABLE         87
+#define TK_ADD            88
+#define TK_COLUMN         89
+#define TK_MODIFY         90
+#define TK_RENAME         91
+#define TK_TAG            92
+#define TK_SET            93
+#define TK_NK_EQ          94
+#define TK_USING          95
+#define TK_TAGS           96
+#define TK_COMMENT        97
+#define TK_BOOL           98
+#define TK_TINYINT        99
+#define TK_SMALLINT       100
+#define TK_INT            101
+#define TK_INTEGER        102
+#define TK_BIGINT         103
+#define TK_FLOAT          104
+#define TK_DOUBLE         105
+#define TK_BINARY         106
+#define TK_TIMESTAMP      107
+#define TK_NCHAR          108
+#define TK_UNSIGNED       109
+#define TK_JSON           110
+#define TK_VARCHAR        111
+#define TK_MEDIUMBLOB     112
+#define TK_BLOB           113
+#define TK_VARBINARY      114
+#define TK_DECIMAL        115
+#define TK_MAX_DELAY      116
+#define TK_WATERMARK      117
+#define TK_ROLLUP         118
+#define TK_TTL            119
+#define TK_SMA            120
+#define TK_FIRST          121
+#define TK_LAST           122
+#define TK_SHOW           123
+#define TK_DATABASES      124
+#define TK_TABLES         125
+#define TK_STABLES        126
+#define TK_MNODES         127
+#define TK_MODULES        128
+#define TK_QNODES         129
+#define TK_FUNCTIONS      130
+#define TK_INDEXES        131
+#define TK_ACCOUNTS       132
+#define TK_APPS           133
+#define TK_CONNECTIONS    134
+#define TK_LICENCE        135
+#define TK_GRANTS         136
+#define TK_QUERIES        137
+#define TK_SCORES         138
+#define TK_TOPICS         139
+#define TK_VARIABLES      140
+#define TK_BNODES         141
+#define TK_SNODES         142
+#define TK_CLUSTER        143
+#define TK_TRANSACTIONS   144
+#define TK_DISTRIBUTED    145
+#define TK_CONSUMERS      146
+#define TK_SUBSCRIPTIONS  147
+#define TK_LIKE           148
+#define TK_INDEX          149
+#define TK_FUNCTION       150
+#define TK_INTERVAL       151
+#define TK_TOPIC          152
+#define TK_AS             153
+#define TK_WITH           154
+#define TK_META           155
+#define TK_CONSUMER       156
+#define TK_GROUP          157
+#define TK_DESC           158
+#define TK_DESCRIBE       159
+#define TK_RESET          160
+#define TK_QUERY          161
+#define TK_CACHE          162
+#define TK_EXPLAIN        163
+#define TK_ANALYZE        164
+#define TK_VERBOSE        165
+#define TK_NK_BOOL        166
+#define TK_RATIO          167
+#define TK_NK_FLOAT       168
+#define TK_COMPACT        169
+#define TK_VNODES         170
+#define TK_IN             171
+#define TK_OUTPUTTYPE     172
+#define TK_AGGREGATE      173
+#define TK_BUFSIZE        174
+#define TK_STREAM         175
+#define TK_INTO           176
+#define TK_TRIGGER        177
+#define TK_AT_ONCE        178
+#define TK_WINDOW_CLOSE   179
+#define TK_IGNORE         180
+#define TK_EXPIRED        181
+#define TK_KILL           182
+#define TK_CONNECTION     183
+#define TK_TRANSACTION    184
+#define TK_BALANCE        185
+#define TK_VGROUP         186
+#define TK_MERGE          187
+#define TK_REDISTRIBUTE   188
+#define TK_SPLIT          189
+#define TK_SYNCDB         190
+#define TK_DELETE         191
+#define TK_INSERT         192
+#define TK_NULL           193
+#define TK_NK_QUESTION    194
+#define TK_NK_ARROW       195
+#define TK_ROWTS          196
+#define TK_TBNAME         197
+#define TK_QSTARTTS       198
+#define TK_QENDTS         199
+#define TK_WSTARTTS       200
+#define TK_WENDTS         201
+#define TK_WDURATION      202
+#define TK_CAST           203
+#define TK_NOW            204
+#define TK_TODAY          205
+#define TK_TIMEZONE       206
+#define TK_CLIENT_VERSION 207
+#define TK_SERVER_VERSION 208
+#define TK_SERVER_STATUS  209
+#define TK_CURRENT_USER   210
+#define TK_COUNT          211
+#define TK_LAST_ROW       212
+#define TK_BETWEEN        213
+#define TK_IS             214
+#define TK_NK_LT          215
+#define TK_NK_GT          216
+#define TK_NK_LE          217
+#define TK_NK_GE          218
+#define TK_NK_NE          219
+#define TK_MATCH          220
+#define TK_NMATCH         221
+#define TK_CONTAINS       222
+#define TK_JOIN           223
+#define TK_INNER          224
+#define TK_SELECT         225
+#define TK_DISTINCT       226
+#define TK_WHERE          227
+#define TK_PARTITION      228
+#define TK_BY             229
+#define TK_SESSION        230
+#define TK_STATE_WINDOW   231
+#define TK_SLIDING        232
+#define TK_FILL           233
+#define TK_VALUE          234
+#define TK_NONE           235
+#define TK_PREV           236
+#define TK_LINEAR         237
+#define TK_NEXT           238
+#define TK_HAVING         239
+#define TK_RANGE          240
+#define TK_EVERY          241
+#define TK_ORDER          242
+#define TK_SLIMIT         243
+#define TK_SOFFSET        244
+#define TK_LIMIT          245
+#define TK_OFFSET         246
+#define TK_ASC            247
+#define TK_NULLS          248
+#define TK_ID             249
+#define TK_NK_BITNOT      250
+#define TK_VALUES         251
+#define TK_IMPORT         252
+#define TK_NK_SEMI        253
+#define TK_FILE           254

 #define TK_NK_SPACE   300
 #define TK_NK_COMMENT 301
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@ -36,6 +36,7 @@ typedef struct SReadHandle {
  void*   vnode;
  void*   mnd;
  SMsgCb* pMsgCb;
+  int64_t version;
  bool    initMetaReader;
  bool    initTableReader;
  bool    initTqReader;
@ -109,7 +110,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
 * @param tversion
 * @return
 */
-int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
+int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
                                      int32_t* tversion);

 /**
@ -176,7 +177,7 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);

 int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);

-int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset);
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset);

 int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);

--- a/include/libs/nodes/cmdnodes.h
+++ b/include/libs/nodes/cmdnodes.h
@ -103,6 +103,11 @@ typedef struct SFlushDatabaseStmt {
  char      dbName[TSDB_DB_NAME_LEN];
 } SFlushDatabaseStmt;

+typedef struct STrimDatabaseStmt {
+  ENodeType type;
+  char      dbName[TSDB_DB_NAME_LEN];
+} STrimDatabaseStmt;
+
 typedef struct STableOptions {
  ENodeType  type;
  bool       commentNull;
--- a/include/libs/nodes/nodes.h
+++ b/include/libs/nodes/nodes.h
@ -112,6 +112,7 @@ typedef enum ENodeType {
  QUERY_NODE_DROP_DATABASE_STMT,
  QUERY_NODE_ALTER_DATABASE_STMT,
  QUERY_NODE_FLUSH_DATABASE_STMT,
+  QUERY_NODE_TRIM_DATABASE_STMT,
  QUERY_NODE_CREATE_TABLE_STMT,
  QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
  QUERY_NODE_CREATE_MULTI_TABLE_STMT,
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@ -250,6 +250,7 @@ typedef struct SSelectStmt {
  SLimitNode* pSlimit;
  char        stmtName[TSDB_TABLE_NAME_LEN];
  uint8_t     precision;
+  int32_t     selectFuncNum;
  bool        isEmptyResult;
  bool        isTimeLineResult;
  bool        hasAggFuncs;
@ -257,6 +258,7 @@ typedef struct SSelectStmt {
  bool        hasIndefiniteRowsFunc;
  bool        hasSelectFunc;
  bool        hasSelectValFunc;
+  bool        hasOtherVectorFunc;
  bool        hasUniqueFunc;
  bool        hasTailFunc;
  bool        hasInterpFunc;
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@ -25,7 +25,7 @@ extern "C" {

 typedef struct SFilterInfo SFilterInfo;

-int32_t scalarGetOperatorResultType(SDataType left, SDataType right, EOperatorType op, SDataType* pRes);
+int32_t scalarGetOperatorResultType(SOperatorNode* pOp);

 /*
 pNode will be freed in API;
@ -42,7 +42,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type);
 int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);

 int32_t vectorGetConvertType(int32_t type1, int32_t type2);
-int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut);
+int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* overflow);

 /* Math functions */
 int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@ -27,7 +27,7 @@ extern "C" {

 #define TAOS_CONN_SERVER 0
 #define TAOS_CONN_CLIENT 1
-#define IsReq(pMsg) (pMsg->msgType & 1U)
+#define IsReq(pMsg)      (pMsg->msgType & 1U)

 extern int32_t tsRpcHeadSize;

--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@ -45,7 +45,6 @@ extern "C" {
 #define WAL_MAGIC        0xFAFBFCFDULL

 typedef enum {
-  TAOS_WAL_NOLOG = 0,
  TAOS_WAL_WRITE = 1,
  TAOS_WAL_FSYNC = 2,
 } EWalType;
@ -74,7 +73,7 @@ typedef struct {
  int8_t   isWeek;
  uint64_t seqNum;
  uint64_t term;
-} SSyncLogMeta;
+} SWalSyncInfo;

 typedef struct {
  int8_t  protoVer;
@ -84,7 +83,7 @@ typedef struct {
  int64_t ingestTs;  // not implemented

  // sync meta
-  SSyncLogMeta syncMeta;
+  SWalSyncInfo syncMeta;

  char body[];
 } SWalCont;
@ -124,6 +123,7 @@ typedef struct SWal {
 typedef struct {
  int8_t scanUncommited;
  int8_t scanMeta;
+  int8_t enableRef;
 } SWalFilterCond;

 typedef struct {
@ -133,6 +133,7 @@ typedef struct {
  int64_t        curFileFirstVer;
  int64_t        curVersion;
  int64_t        capacity;
+  int8_t         curInvalid;
  TdThreadMutex  mutex;
  SWalFilterCond cond;
  SWalCkHead    *pHead;
@ -147,11 +148,22 @@ SWal   *walOpen(const char *path, SWalCfg *pCfg);
 int32_t walAlter(SWal *, SWalCfg *pCfg);
 void    walClose(SWal *);

-// write
-int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body,
-                             int32_t bodyLen);
+// write interfaces
+
+// By assigning index by the caller, wal gurantees linearizability
 int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
-void    walFsync(SWal *, bool force);
+int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
+                             int32_t bodyLen);
+
+// This interface assign version automatically and return to caller.
+// When using this interface with concurrent writes,
+// wal will write all logs atomically,
+// but not sure which one will be actually write first,
+// and then the unique index of successful writen is returned.
+// -1 will be returned for failed writes
+int64_t walAppendLog(SWal *, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
+
+void walFsync(SWal *, bool force);

 // apis for lifecycle management
 int32_t walCommit(SWal *, int64_t ver);
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@ -584,6 +584,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_INTERP_CLAUSE     TAOS_DEF_ERROR_CODE(0, 0x265D)
 #define TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN      TAOS_DEF_ERROR_CODE(0, 0x265E)
 #define TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE TAOS_DEF_ERROR_CODE(0, 0x265F)
+#define TSDB_CODE_PAR_INVALID_SMA_INDEX         TAOS_DEF_ERROR_CODE(0, 0x265C)

 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@ -331,12 +331,12 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_STRICT_OFF              0
 #define TSDB_DB_STRICT_ON               1
 #define TSDB_DEFAULT_DB_STRICT          0
-#define TSDB_MIN_DB_CACHE_LAST_ROW      0
-#define TSDB_MAX_DB_CACHE_LAST_ROW      3
-#define TSDB_DEFAULT_CACHE_LAST_ROW     0
-#define TSDB_MIN_DB_LAST_ROW_MEM        1  // MB
-#define TSDB_MAX_DB_LAST_ROW_MEM        65536
-#define TSDB_DEFAULT_LAST_ROW_MEM       1
+#define TSDB_MIN_DB_CACHE_LAST          0
+#define TSDB_MAX_DB_CACHE_LAST          3
+#define TSDB_DEFAULT_CACHE_LAST         0
+#define TSDB_MIN_DB_CACHE_LAST_SIZE     1  // MB
+#define TSDB_MAX_DB_CACHE_LAST_SIZE     65536
+#define TSDB_DEFAULT_CACHE_LAST_SIZE    1
 #define TSDB_DB_STREAM_MODE_OFF         0
 #define TSDB_DB_STREAM_MODE_ON          1
 #define TSDB_DEFAULT_DB_STREAM_MODE     0
--- a/include/util/tlockfree.h
+++ b/include/util/tlockfree.h
@ -69,13 +69,14 @@ typedef void (*_ref_fn_t)(const void *pObj);
 #define T_REF_VAL_GET(x) (x)->_ref.val

 // single writer multiple reader lock
-typedef volatile int32_t SRWLatch;
+typedef volatile int64_t SRWLatch;

-void taosInitRWLatch(SRWLatch *pLatch);
-void taosWLockLatch(SRWLatch *pLatch);
-void taosWUnLockLatch(SRWLatch *pLatch);
-void taosRLockLatch(SRWLatch *pLatch);
-void taosRUnLockLatch(SRWLatch *pLatch);
+void    taosInitRWLatch(SRWLatch *pLatch);
+void    taosInitReentrantRWLatch(SRWLatch *pLatch);
+void    taosWLockLatch(SRWLatch *pLatch);
+void    taosWUnLockLatch(SRWLatch *pLatch);
+void    taosRLockLatch(SRWLatch *pLatch);
+void    taosRUnLockLatch(SRWLatch *pLatch);
 int32_t taosWTryLockLatch(SRWLatch *pLatch);

 // copy on read
--- a/packaging/tools/make_install.sh
+++ b/packaging/tools/make_install.sh
@ -305,18 +305,23 @@ function install_lib() {
      ${install_main_dir}/driver &&
      ${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber}

-    ${csudo}cp ${binary_dir}/build/lib/libtaosws.so \
-      ${install_main_dir}/driver &&
-      ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so
-
    ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
    ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
-    ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
-
    if [ -d "${lib64_link_dir}" ]; then
-      ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1
-      ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so
-      ${csudo}ln -sf ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
+        ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1
+        ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so
+    fi
+
+    if [ -f ${binary_dir}/build/lib/libtaosws.so ]; then
+        ${csudo}cp ${binary_dir}/build/lib/libtaosws.so \
+            ${install_main_dir}/driver &&
+            ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||:
+
+        ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
+
+        if [ -d "${lib64_link_dir}" ]; then
+            ${csudo}ln -sf ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
+        fi
    fi
  else
    ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \
@ -357,26 +362,26 @@ function install_header() {

  if [ "$osType" != "Darwin" ]; then
    ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
+    ${csudo}rm -f ${inc_link_dir}/taosws.h ||:
    ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
-    ${csudo}rm -f ${inc_link_dir}/taosws.h || :
-
-    ${csudo}cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taosdef.h ${source_dir}/src/inc/taoserror.h \
      ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*

-    ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h
+    if [ -f ${binary_dir}/build/include/taosws.h ]; then
+      ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||:
+      ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||:
+    fi

    ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
    ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
    ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
    ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h

-    ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || :
  else
    ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
      ${install_main_dir}/include ||
      ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
        ${install_main_2_dir}/include &&
-      ${csudo}chmod 644 ${install_main_dir}/include/* ||
+      ${csudo}chmod 644 ${install_main_dir}/include/* ||:
      ${csudo}chmod 644 ${install_main_2_dir}/include/*
  fi
 }
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@ -222,8 +222,8 @@ typedef struct SRequestObj {
  SArray*              tableList;
  SQueryExecMetric     metric;
  SRequestSendRecvBody body;
-  bool                 stableQuery;
-  bool                 validateOnly;
+  bool                 stableQuery;  // todo refactor
+  bool                 validateOnly; // todo refactor

  bool     killed;
  uint32_t prevCode;  // previous error code: todo refactor, add update flag for catalog
@ -247,9 +247,9 @@ void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
 int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
 void    syncCatalogFn(SMetaData* pResult, void* param, int32_t code);

-SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly);
+SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly);
 TAOS_RES*    taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
-void         taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
+void         taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);

 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
  SMqRspObj* msg = (SMqRspObj*)res;
@ -297,7 +297,7 @@ int32_t  releaseTscObj(int64_t rid);

 uint64_t generateRequestId();

-void*        createRequest(STscObj* pObj, int32_t type);
+void*        createRequest(uint64_t connId, int32_t type);
 void         destroyRequest(SRequestObj* pRequest);
 SRequestObj* acquireRequest(int64_t rid);
 int32_t      releaseRequest(int64_t rid);
@ -318,13 +318,13 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
 STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                               uint16_t port, int connType);

-SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly);
+SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly);

 int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);

 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);

-int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
+int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql, SRequestObj** pRequest);

 void taos_close_internal(void* taos);

--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@ -49,13 +49,8 @@ void cleanupTscQhandle() {
  // destroy handle
  taosCleanUpScheduler(tscQhandle);
 }
-static int32_t registerRequest(SRequestObj *pRequest) {
-  STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id);
-  if (NULL == pTscObj) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return terrno;
-  }

+static int32_t registerRequest(SRequestObj *pRequest, STscObj* pTscObj) {
  // connection has been released already, abort creating request.
  pRequest->self = taosAddRef(clientReqRefPool, pRequest);

@ -246,29 +241,34 @@ STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientCon

 int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); }

-void *createRequest(STscObj *pObj, int32_t type) {
-  assert(pObj != NULL);
-
+void *createRequest(uint64_t connId, int32_t type) {
  SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj));
  if (NULL == pRequest) {
    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
    return NULL;
  }

+  STscObj* pTscObj = acquireTscObj(connId);
+  if (pTscObj == NULL) {
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
+    return NULL;
+  }
+
  pRequest->resType = RES_TYPE__QUERY;
-  pRequest->pDb = getDbOfConnection(pObj);
  pRequest->requestId = generateRequestId();
  pRequest->metric.start = taosGetTimestampUs();

  pRequest->body.resInfo.convertUcs4 = true;  // convert ucs4 by default
-
  pRequest->type = type;
-  pRequest->pTscObj = pObj;
+
+  pRequest->pDb = getDbOfConnection(pTscObj);
+  pRequest->pTscObj = pTscObj;
+
  pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
  pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE;
  tsem_init(&pRequest->body.rspSem, 0, 0);

-  if (registerRequest(pRequest)) {
+  if (registerRequest(pRequest, pTscObj)) {
    doDestroyRequest(pRequest);
    return NULL;
  }
@ -327,8 +327,8 @@ void doDestroyRequest(void *p) {
  if (pRequest->self) {
    deregisterRequest(pRequest);
  }
-  taosMemoryFree(pRequest);

+  taosMemoryFree(pRequest);
  tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest);
 }

@ -338,7 +338,6 @@ void destroyRequest(SRequestObj *pRequest) {
  }

  taos_stop_query(pRequest);
-
  removeRequest(pRequest->self);
 }

--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@ -264,14 +264,11 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {

 static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
  static int32_t emptyRspNum = 0;
-  if (code != 0) {
-    taosMemoryFreeClear(param);
-    return -1;
-  }
-
  char             *key = (char *)param;
  SClientHbBatchRsp pRsp = {0};
-  tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
+  if (TSDB_CODE_SUCCESS == code) {
+    tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
+  }
  
  int32_t rspNum = taosArrayGetSize(pRsp.rsps);

@ -288,6 +285,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {

  taosMemoryFreeClear(param);

+  if (code != 0) {
+    (*pInst)->onlineDnodes = 0;
+  }
+
  if (rspNum) {
    tscDebug("hb got %d rsp, %d empty rsp received before", rspNum,
             atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0));
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@ -148,29 +148,50 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
  return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
 }

-int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) {
-  *pRequest = createRequest(pTscObj, TSDB_SQL_SELECT);
+int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql,
+                     SRequestObj** pRequest) {
+  *pRequest = createRequest(connId, TSDB_SQL_SELECT);
  if (*pRequest == NULL) {
-    tscError("failed to malloc sqlObj");
+    tscError("failed to malloc sqlObj, %s", sql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

  (*pRequest)->sqlstr = taosMemoryMalloc(sqlLen + 1);
  if ((*pRequest)->sqlstr == NULL) {
-    tscError("0x%" PRIx64 " failed to prepare sql string buffer", (*pRequest)->self);
-    (*pRequest)->msgBuf = strdup("failed to prepare sql string buffer");
+    tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql);
+    destroyRequest(*pRequest);
+    *pRequest = NULL;
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

  strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen);
  (*pRequest)->sqlstr[sqlLen] = 0;
  (*pRequest)->sqlLen = sqlLen;
+  (*pRequest)->validateOnly = validateSql;

+  if (param == NULL) {
+    SSyncQueryParam* pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+    if (pParam == NULL) {
+      destroyRequest(*pRequest);
+      *pRequest = NULL;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+
+    tsem_init(&pParam->sem, 0, 0);
+    pParam->pRequest = (*pRequest);
+    param = pParam;
+  }
+
+  (*pRequest)->body.param = param;
+
+  STscObj* pTscObj = (*pRequest)->pTscObj;
  if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
                  sizeof((*pRequest)->self))) {
+    tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self,
+             (*pRequest)->requestId, pTscObj->id, sql);
+
    destroyRequest(*pRequest);
    *pRequest = NULL;
-    tscError("put request to request hash failed");
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

@ -325,11 +346,14 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
  if (pInfo->pQnodeList) {
    taosArrayDestroy(pInfo->pQnodeList);
    pInfo->pQnodeList = NULL;
+    tscDebug("QnodeList cleared in cluster 0x%" PRIx64, pInfo->clusterId);
  }

  if (pNodeList) {
    pInfo->pQnodeList = taosArrayDup(pNodeList);
    taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad);
+    tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId,
+             taosArrayGetSize(pInfo->pQnodeList));
  }
  taosThreadMutexUnlock(&pInfo->qnodeMutex);

@ -627,22 +651,22 @@ _return:
 int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
  void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;

-  SExecResult     res = {0};
+  SExecResult      res = {0};
  SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
                           .requestId = pRequest->requestId,
                           .requestObjRefId = pRequest->self};
  SSchedulerReq    req = {
-    .syncReq = true,
-    .pConn = &conn,
-    .pNodeList = pNodeList,
-    .pDag = pDag,
-    .sql = pRequest->sqlstr,
-    .startTs = pRequest->metric.start,
-    .execFp = NULL,
-    .cbParam = NULL,
-    .chkKillFp = chkRequestKilled,
-    .chkKillParam = (void*)pRequest->self,
-    .pExecRes = &res,
+         .syncReq = true,
+         .pConn = &conn,
+         .pNodeList = pNodeList,
+         .pDag = pDag,
+         .sql = pRequest->sqlstr,
+         .startTs = pRequest->metric.start,
+         .execFp = NULL,
+         .cbParam = NULL,
+         .chkKillFp = chkRequestKilled,
+         .chkKillParam = (void*)pRequest->self,
+         .pExecRes = &res,
  };

  int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob);
@ -756,7 +780,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
    return code;
  }

-  SEpSet         epset = getEpSet_s(&pAppInfo->mgmtEp);
+  SEpSet       epset = getEpSet_s(&pAppInfo->mgmtEp);
  SExecResult* pRes = &pRequest->body.resInfo.execRes;

  switch (pRes->msgType) {
@ -880,18 +904,16 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
  return pRequest;
 }

-SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) {
+SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) {
  SRequestObj* pRequest = NULL;
  SQuery*      pQuery = NULL;

-  int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
+  int32_t code = buildRequest(connId, sql, sqlLen, NULL, validateOnly, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    terrno = code;
    return NULL;
  }

-  pRequest->validateOnly = validateOnly;
-
  code = parseSql(pRequest, false, &pQuery, NULL);
  if (code != TSDB_CODE_SUCCESS) {
    pRequest->code = code;
@ -944,17 +966,17 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
        SRequestConnInfo conn = {
            .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
        SSchedulerReq req = {
-          .syncReq = false,
-          .pConn = &conn,
-          .pNodeList = pNodeList,
-          .pDag = pDag,
-          .sql = pRequest->sqlstr,
-          .startTs = pRequest->metric.start,
-          .execFp = schedulerExecCb,
-          .cbParam = pRequest,
-          .chkKillFp = chkRequestKilled,
-          .chkKillParam = (void*)pRequest->self,
-          .pExecRes = NULL,
+            .syncReq = false,
+            .pConn = &conn,
+            .pNodeList = pNodeList,
+            .pDag = pDag,
+            .sql = pRequest->sqlstr,
+            .startTs = pRequest->metric.start,
+            .execFp = schedulerExecCb,
+            .cbParam = pRequest,
+            .chkKillFp = chkRequestKilled,
+            .chkKillParam = (void*)pRequest->self,
+            .pExecRes = NULL,
        };
        code = schedulerExecJob(&req, &pRequest->body.queryJob);
        taosArrayDestroy(pNodeList);
@ -973,6 +995,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
      break;
    default:
+      pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
      break;
  }

@ -1042,19 +1065,20 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) {
  return TSDB_CODE_SUCCESS;
 }

-SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) {
+//  todo remove it soon
+SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) {
  SRequestObj* pRequest = NULL;
  int32_t      retryNum = 0;
  int32_t      code = 0;

  do {
    destroyRequest(pRequest);
-    pRequest = launchQuery(pTscObj, sql, sqlLen, validateOnly);
+    pRequest = launchQuery(connId, sql, sqlLen, validateOnly);
    if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
      break;
    }

-    code = refreshMeta(pTscObj, pRequest);
+    code = refreshMeta(pRequest->pTscObj, pRequest);
    if (code) {
      pRequest->code = code;
      break;
@ -1062,7 +1086,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool valid
  } while (retryNum++ < REQUEST_TOTAL_EXEC_TIMES);

  if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
-    removeMeta(pTscObj, pRequest->tableList);
+    removeMeta(pRequest->pTscObj, pRequest->tableList);
  }

  return pRequest;
@ -1117,7 +1141,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
    return pTscObj;
  }

-  SRequestObj* pRequest = createRequest(pTscObj, TDMT_MND_CONNECT);
+  SRequestObj* pRequest = createRequest(pTscObj->id, TDMT_MND_CONNECT);
  if (pRequest == NULL) {
    destroyTscObj(pTscObj);
    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -1395,9 +1419,9 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
    }

    SReqResultInfo* pResInfo = &pRequest->body.resInfo;
-    SSchedulerReq req = {
-      .syncReq = true,
-      .pFetchRes = (void**)&pResInfo->pData,
+    SSchedulerReq   req = {
+          .syncReq = true,
+          .pFetchRes = (void**)&pResInfo->pData,
    };
    pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
    if (pRequest->code != TSDB_CODE_SUCCESS) {
@ -1444,25 +1468,24 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
      return NULL;
    }

-    SSyncQueryParam* pParam = pRequest->body.param;
-    if (NULL == pParam) {
-      pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
-      tsem_init(&pParam->sem, 0, 0);
-    }
-
    // convert ucs4 to native multi-bytes string
    pResultInfo->convertUcs4 = convertUcs4;

+    SSyncQueryParam* pParam = pRequest->body.param;
    taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
    tsem_wait(&pParam->sem);
  }

-  if (pRequest->code == TSDB_CODE_SUCCESS && pResultInfo->numOfRows > 0 && setupOneRowPtr) {
-    doSetOneRowPtr(pResultInfo);
-    pResultInfo->current += 1;
-  }
+  if (pResultInfo->numOfRows == 0  || pRequest->code != TSDB_CODE_SUCCESS) {
+    return NULL;
+  } else {
+    if (setupOneRowPtr) {
+      doSetOneRowPtr(pResultInfo);
+      pResultInfo->current += 1;
+    }

-  return pResultInfo->row;
+    return pResultInfo->row;
+  }
 }

 static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
@ -2024,22 +2047,9 @@ void syncQueryFn(void* param, void* res, int32_t code) {
  tsem_post(&pParam->sem);
 }

-void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
-  if (NULL == taos) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    fp(param, NULL, terrno);
-    return;
-  }
-
-  int64_t  rid = *(int64_t*)taos;
-  STscObj* pTscObj = acquireTscObj(rid);
-  if (pTscObj == NULL || sql == NULL || NULL == fp) {
+void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
+  if (sql == NULL || NULL == fp) {
    terrno = TSDB_CODE_INVALID_PARA;
-    if (pTscObj) {
-      releaseTscObj(rid);
-    } else {
-      terrno = TSDB_CODE_TSC_DISCONNECTED;
-    }
    fp(param, NULL, terrno);
    return;
  }
@ -2048,26 +2058,20 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
  if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
-    releaseTscObj(rid);
-
    fp(param, NULL, terrno);
    return;
  }

  SRequestObj* pRequest = NULL;
-  int32_t      code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
+  int32_t      code = buildRequest(connId, sql, sqlLen, param, validateOnly, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    terrno = code;
-    releaseTscObj(rid);
    fp(param, NULL, terrno);
    return;
  }

-  pRequest->validateOnly = validateOnly;
  pRequest->body.queryFp = fp;
-  pRequest->body.param = param;
  doAsyncQuery(pRequest, false);
-  releaseTscObj(rid);
 }

 TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
@ -2076,36 +2080,22 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
    return NULL;
  }

-  int64_t  rid = *(int64_t*)taos;
-  STscObj* pTscObj = acquireTscObj(rid);
-  if (pTscObj == NULL || sql == NULL) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return NULL;
-  }
-
 #if SYNC_ON_TOP_OF_ASYNC
  SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
  tsem_init(&param->sem, 0, 0);

-  taosAsyncQueryImpl((TAOS*)&rid, sql, syncQueryFn, param, validateOnly);
+  taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
  tsem_wait(&param->sem);
-
-  releaseTscObj(rid);
-
  return param->pRequest;
 #else
  size_t sqlLen = strlen(sql);
  if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
-    releaseTscObj(rid);
    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
    return NULL;
  }

-  TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
-
-  releaseTscObj(rid);
-
+  TAOS_RES* pRes = execQuery(connId, sql, sqlLen, validateOnly);
  return pRes;
 #endif
 }
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@ -479,7 +479,6 @@ void taos_stop_query(TAOS_RES *res) {
  }

  schedulerFreeJob(&pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED);
-
  tscDebug("request %" PRIx64 " killed", pRequest->requestId);
 }

@ -706,7 +705,8 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
 }

 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
-  taosAsyncQueryImpl(taos, sql, fp, param, false);
+  int64_t  connId = *(int64_t*)taos;
+  taosAsyncQueryImpl(connId, sql, fp, param, false);
 }

 int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
@ -915,7 +915,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
    return terrno;
  }

-  int64_t       rid = *(int64_t *)taos;
+  int64_t       connId = *(int64_t *)taos;
  const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024;  // 12MB list
  int32_t       code = 0;
  SRequestObj * pRequest = NULL;
@ -933,12 +933,14 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
    return TSDB_CODE_TSC_INVALID_OPERATION;
  }

-  STscObj *pTscObj = acquireTscObj(rid);
-  if (pTscObj == NULL) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return terrno;
+  char *sql = "taos_load_table_info";
+  code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest);
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    goto _return;
  }

+  STscObj* pTscObj = pRequest->pTscObj;
  code = transferTableNameList(tableNameList, pTscObj->acctId, pTscObj->db, &catalogReq.pTableMeta);
  if (code) {
    goto _return;
@ -950,36 +952,22 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
    goto _return;
  }

-  char *sql = "taos_load_table_info";
-  code = buildRequest(pTscObj, sql, strlen(sql), &pRequest);
-  if (code != TSDB_CODE_SUCCESS) {
-    terrno = code;
-    goto _return;
-  }
-
-  SSyncQueryParam param = {0};
-  tsem_init(&param.sem, 0, 0);
-  param.pRequest = pRequest;
-
  SRequestConnInfo conn = {
      .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};

  conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);

-  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, &param, NULL);
+  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, NULL, NULL);
  if (code) {
    goto _return;
  }

-  tsem_wait(&param.sem);
+  SSyncQueryParam* pParam = pRequest->body.param;
+  tsem_wait(&pParam->sem);

 _return:
-
  taosArrayDestroy(catalogReq.pTableMeta);
  destroyRequest(pRequest);
-
-  releaseTscObj(rid);
-
  return code;
 }

--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@ -2442,22 +2442,15 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
    return NULL;
  }

-  int64_t rid = *(int64_t*)taos;
-  STscObj* pTscObj = acquireTscObj(rid);
-  if (NULL == pTscObj) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    uError("SML:taos_schemaless_insert invalid taos");
-    return NULL;
-  }
-  
-  SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT);
+  SRequestObj* request = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT);
  if(!request){
-    releaseTscObj(rid);
    uError("SML:taos_schemaless_insert error request is null");
    return NULL;
  }

  int    batchs = 0;
+  STscObj* pTscObj = request->pTscObj;
+
  pTscObj->schemalessType = 1;
  SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};

@ -2507,7 +2500,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr

  batchs = ceil(((double)numLines) / LINE_BATCH);
  for (int i = 0; i < batchs; ++i) {
-    SRequestObj* req = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT);
+    SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT);
    if(!req){
      request->code = TSDB_CODE_OUT_OF_MEMORY;
      uError("SML:taos_schemaless_insert error request is null");
@ -2549,6 +2542,5 @@ end:
 //  ((STscObj *)taos)->schemalessType = 0;
  pTscObj->schemalessType = 1;
  uDebug("resultend:%s", request->msgBuf);
-  releaseTscObj(rid);
  return (TAOS_RES*)request;
 }
--- a/source/client/src/clientStmt.c
+++ b/source/client/src/clientStmt.c
@ -5,6 +5,14 @@

 #include "clientStmt.h"

+static int32_t stmtCreateRequest(STscStmt* pStmt) {
+  if (pStmt->exec.pRequest == NULL)  {
+    return buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, &pStmt->exec.pRequest);
+  } else {
+    return TSDB_CODE_SUCCESS;
+  }
+}
+
 int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
  int32_t code = 0;

@ -218,9 +226,7 @@ int32_t stmtParseSql(STscStmt* pStmt) {
      .getExecInfoFn = stmtGetExecInfo,
  };

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb));

@ -533,9 +539,7 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb,
                            pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
@ -626,9 +630,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
@ -874,9 +876,7 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
@ -906,9 +906,7 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
@ -934,10 +932,7 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
-
+  STMT_ERR_RET(stmtCreateRequest(pStmt));
  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
  }
@ -970,9 +965,7 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@ -13,6 +13,7 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include "cJSON.h"
 #include "clientInt.h"
 #include "clientLog.h"
 #include "parser.h"
@ -23,7 +24,6 @@
 #include "tqueue.h"
 #include "tref.h"
 #include "ttimer.h"
-#include "cJSON.h"

 int32_t tmqAskEp(tmq_t* tmq, bool async);

@ -106,8 +106,8 @@ struct tmq_t {
  tsem_t rspSem;
 };

-struct tmq_raw_data{
-  void    *raw_meta;
+struct tmq_raw_data {
+  void*   raw_meta;
  int32_t raw_meta_len;
  int16_t raw_meta_type;
 };
@ -901,6 +901,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {

  tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t));
  if (pTmq == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    tscError("consumer %ld setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId);
    return NULL;
  }

@ -917,6 +919,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
  pTmq->delayedTask = taosOpenQueue();

  if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    tscError("consumer %ld setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId);
    goto FAIL;
  }

@ -943,16 +947,20 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {

  // init semaphore
  if (tsem_init(&pTmq->rspSem, 0, 0) != 0) {
+    tscError("consumer %ld setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId);
    goto FAIL;
  }

  // init connection
  pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ);
  if (pTmq->pTscObj == NULL) {
+    tscError("consumer %ld setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId);
    tsem_destroy(&pTmq->rspSem);
    goto FAIL;
  }

+  tscInfo("consumer %ld is setup, consumer group %s", pTmq->consumerId, pTmq->groupId);
+
  return pTmq;

 FAIL:
@ -1194,10 +1202,10 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
      for (int32_t j = 0; j < vgNumCur; j++) {
        SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j);
        sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId);
-        char buf[50];
-        tFormatOffset(buf, 50, &pVgCur->currentOffsetNew);
-        tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId,
-                 vgKey, buf);
+        char buf[80];
+        tFormatOffset(buf, 80, &pVgCur->currentOffsetNew);
+        tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch,
+                 pVgCur->vgId, vgKey, buf);
        taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal));
      }
    }
@ -1564,7 +1572,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
      int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
      if (vgStatus != TMQ_VG_STATUS__IDLE) {
        int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
-        tscTrace("consumer:%" PRId64 ", epoch %d skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt);
+        tscTrace("consumer:%" PRId64 ", epoch %d skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId,
+                 vgSkipCnt);
        continue;
        /*if (vgSkipCnt < 10000) continue;*/
 #if 0
@ -1620,8 +1629,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {

      char offsetFormatBuf[80];
      tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew);
-      tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, tmq->consumerId,
-               pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId);
+      tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64,
+               tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId);
      /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/
      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
      pVg->pollCnt++;
@ -1669,7 +1678,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
      int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
      if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
        SMqClientVg* pVg = pollRspWrapper->vgHandle;
-        /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
+        /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset,
+         * rspMsg->msg.rspOffset);*/
        pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset;
        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
        if (pollRspWrapper->dataRsp.blockNum == 0) {
@ -1691,7 +1701,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
      int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
      if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
        SMqClientVg* pVg = pollRspWrapper->vgHandle;
-        /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
+        /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset,
+         * rspMsg->msg.rspOffset);*/
        pVg->currentOffsetNew.version = pollRspWrapper->metaRsp.rspOffset;
        pVg->currentOffsetNew.type = TMQ_OFFSET__LOG;
        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
@ -2013,7 +2024,7 @@ static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){
  }
  string = cJSON_PrintUnformatted(json);

-end:
+  end:
  cJSON_Delete(json);
  tFreeSMAltertbReq(&req);
  return string;
@ -2036,7 +2047,7 @@ static char *processCreateStb(SMqMetaRsp *metaRsp){
  tDecoderClear(&coder);
  return string;

-_err:
+  _err:
  tDecoderClear(&coder);
  return string;
 }
@ -2391,13 +2402,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){
  int32_t code = TSDB_CODE_SUCCESS;
  SRequestObj* pRequest = NULL;

-  STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
-  if (NULL == pTscObj) {
-    code = TSDB_CODE_TSC_DISCONNECTED;
-    goto end;
-  }
-
-  code = buildRequest(pTscObj, "", 0, &pRequest);
+  code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    goto end;
  }
@ -2438,6 +2443,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){
  pReq.source = TD_REQ_FROM_TAOX;
  pReq.igExists = true;

+  STscObj* pTscObj = pRequest->pTscObj;
  SName tableName;
  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);

@ -2476,13 +2482,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){
  int32_t code = TSDB_CODE_SUCCESS;
  SRequestObj* pRequest = NULL;

-  STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
-  if (NULL == pTscObj) {
-    code = TSDB_CODE_TSC_DISCONNECTED;
-    goto end;
-  }
-
-  code = buildRequest(pTscObj, "", 0, &pRequest);
+  code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    goto end;
  }
@ -2504,6 +2504,8 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){
  pReq.igNotExists = true;
  pReq.source = TD_REQ_FROM_TAOX;
  pReq.suid = req.suid;
+
+  STscObj* pTscObj = pRequest->pTscObj;
  SName tableName;
  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);

@ -2552,18 +2554,13 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){
  SRequestObj        *pRequest        = NULL;
  SQuery             *pQuery          = NULL;
  SHashObj           *pVgroupHashmap  = NULL;
-  STscObj            *pTscObj         = acquireTscObj(*(int64_t *)taos);

-  if (NULL == pTscObj) {
-    code = TSDB_CODE_TSC_DISCONNECTED;
-    goto end;
-  }
-
-  code = buildRequest(pTscObj, "", 0, &pRequest);
+  code = buildRequest(*(int64_t*) taos, "", 0, NULL, false, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    goto end;
  }

+
  if(!pRequest->pDb){
    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
    goto end;
@ -2577,6 +2574,8 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){
    goto end;
  }

+  STscObj* pTscObj = pRequest->pTscObj;
+
  SVCreateTbReq     *pCreateReq = NULL;
  SCatalog* pCatalog = NULL;
  code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
@ -2669,14 +2668,8 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){
  SRequestObj        *pRequest        = NULL;
  SQuery             *pQuery          = NULL;
  SHashObj           *pVgroupHashmap  = NULL;
-  STscObj            *pTscObj         = acquireTscObj(*(int64_t *)taos);

-  if (NULL == pTscObj) {
-    code = TSDB_CODE_TSC_DISCONNECTED;
-    goto end;
-  }
-
-  code = buildRequest(pTscObj, "", 0, &pRequest);
+  code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    goto end;
  }
@ -2694,6 +2687,8 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){
    goto end;
  }

+  STscObj* pTscObj = pRequest->pTscObj;
+
  SVDropTbReq     *pDropReq = NULL;
  SCatalog        *pCatalog = NULL;
  code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
@ -2774,14 +2769,8 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){
  SQuery             *pQuery          = NULL;
  SArray             *pArray          = NULL;
  SVgDataBlocks      *pVgData         = NULL;
-  STscObj            *pTscObj         = acquireTscObj(*(int64_t *)taos);

-  if (NULL == pTscObj) {
-    code = TSDB_CODE_TSC_DISCONNECTED;
-    goto end;
-  }
-
-  code = buildRequest(pTscObj, "", 0, &pRequest);
+  code = buildRequest(*(int64_t*) taos, "", 0, NULL, false, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    goto end;
  }
@ -2804,6 +2793,7 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){
    goto end;
  }

+  STscObj*  pTscObj = pRequest->pTscObj;
  SCatalog        *pCatalog = NULL;
  code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
  if (code != TSDB_CODE_SUCCESS) {
@ -2863,7 +2853,7 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){
  pArray  = NULL;
  code    = pRequest->code;

-end:
+  end:
  taosArrayDestroy(pArray);
  if(pVgData) taosMemoryFreeClear(pVgData->pData);
  taosMemoryFreeClear(pVgData);
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@ -252,7 +252,7 @@ static const SSysTableMeta infosMeta[] = {
    {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
    {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
    {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
-    {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
+    // {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
    {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
    {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
    {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@ -463,6 +463,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3

  pDst->info = pBlock->info;
  pDst->info.rows = 0;
+  pDst->info.capacity = 0;
  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData  colInfo = {0};
@ -1746,7 +1747,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
    for (int32_t k = 0; k < colNum; k++) {
      SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
      void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-      if (colDataIsNull(pColInfoData, rows, j, NULL)) {
+      if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) {
        len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
        if (len >= size -1) return dumpBuf;
        continue;
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@ -55,12 +55,11 @@ int32_t tsNumOfMnodeQueryThreads = 2;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 2;
-int32_t tsNumOfVnodeFetchThreads = 1;
+int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeWriteThreads = 2;
 int32_t tsNumOfVnodeSyncThreads = 2;
-int32_t tsNumOfVnodeMergeThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 2;
-int32_t tsNumOfQnodeFetchThreads = 1;
+int32_t tsNumOfQnodeFetchThreads = 4;
 int32_t tsNumOfSnodeSharedThreads = 2;
 int32_t tsNumOfSnodeUniqueThreads = 2;

@ -106,11 +105,6 @@ int32_t tsCompressMsgSize = -1;
 */
 int32_t tsCompressColData = -1;

-/*
- * denote if 3.0 query pattern compatible for 2.0
- */
-int32_t tsCompatibleModel = 1;
-
 // count/hyperloglog function always return values in case of all NULL data or Empty data set.
 int32_t tsCountAlwaysReturnValue = 1;

@ -190,7 +184,6 @@ int32_t tsMqRebalanceInterval = 2;
 int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushInterval = 60;

-
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
  tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
  tsDiskCfg[index].level = level;
@ -292,15 +285,14 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) {
  if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1;
  if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1;
  return 0;
 }

@ -308,7 +300,6 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
  if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0) != 0) return -1;
-  if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1;
@ -417,8 +408,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
  tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4);
  if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfVnodeQueryThreads = tsNumOfCores / 2;
-  tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 1);
+  tsNumOfVnodeQueryThreads = tsNumOfCores / 4;
+  tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 2);
  if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1;

  tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 1, 1);
@ -428,19 +419,16 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
  tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
  if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfVnodeSyncThreads = tsNumOfCores / 2;
+  tsNumOfVnodeSyncThreads = tsNumOfCores;
  tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1);
  if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfVnodeMergeThreads = tsNumOfCores / 8;
-  tsNumOfVnodeMergeThreads = TRANGE(tsNumOfVnodeMergeThreads, 1, 1);
-  if (cfgAddInt32(pCfg, "numOfVnodeMergeThreads", tsNumOfVnodeMergeThreads, 1, 1024, 0) != 0) return -1;
-
  tsNumOfQnodeQueryThreads = tsNumOfCores / 2;
  tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1);
  if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfQnodeFetchThreads = TRANGE(tsNumOfQnodeFetchThreads, 1, 1);
+  tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
+  tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
  if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;

  tsNumOfSnodeSharedThreads = tsNumOfCores / 4;
@ -470,7 +458,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {

  if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400*365, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 10000, 1) != 0) return -1;

  if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
@ -485,20 +473,18 @@ static void taosSetClientLogCfg(SConfig *pCfg) {
  tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
  tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
  tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
-  cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
-  uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
-  qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
-  rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
  tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
+  uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
  jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
-  idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32;
+  rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
+  qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
+  cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
 }

 static void taosSetServerLogCfg(SConfig *pCfg) {
  dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
  vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32;
  mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32;
-  qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
  wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32;
  sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32;
  tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
@ -603,7 +589,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
  tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
  tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
  tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
-  tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32;
  tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
  tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
  tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
@ -636,7 +621,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
  return 0;
 }

-int32_t taosSetCfg(SConfig *pCfg, char* name) {
+int32_t taosSetCfg(SConfig *pCfg, char *name) {
  int32_t len = strlen(name);
  char    lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
  strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len));
@ -845,8 +830,6 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
        tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
      } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) {
        tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
-      } else if (strcasecmp("numOfVnodeMergeThreads", name) == 0) {
-        tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32;
      } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) {
        tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
      } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) {
@ -1006,7 +989,6 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
  return 0;
 }

-
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
                      const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
  if (tsCfg == NULL) osDefaultInit();
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@ -381,9 +381,11 @@ static int32_t tDeserializeSClientHbRsp(SDecoder *pDecoder, SClientHbRsp *pRsp)
    if (pQnodeNum > 0) {
      pRsp->query->pQnodeList = taosArrayInit(pQnodeNum, sizeof(SQueryNodeLoad));
      if (NULL == pRsp->query->pQnodeList) return -1;
-      SQueryNodeLoad load = {0};
-      if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
-      taosArrayPush(pRsp->query->pQnodeList, &load);
+      for (int32_t i = 0; i < pQnodeNum; ++i) {
+        SQueryNodeLoad load = {0};
+        if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
+        taosArrayPush(pRsp->query->pQnodeList, &load);
+      }
    }
  }

@ -1999,7 +2001,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -2012,7 +2014,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
  if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->schemaless) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1;
@ -2041,7 +2043,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -2054,7 +2056,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
  if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->schemaless) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1;
@ -2096,7 +2098,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -2104,7 +2106,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
  tEndEncode(&encoder);

@ -2122,7 +2124,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -2130,7 +2132,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
  tEndDecode(&decoder);

@ -2645,6 +2647,31 @@ int32_t tDeserializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) {
  return 0;
 }

+int32_t tSerializeSTrimDbReq(void *buf, int32_t bufLen, STrimDbReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSTrimDbReq(void *buf, int32_t bufLen, STrimDbReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
 int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
  SEncoder encoder = {0};
  tEncoderInit(&encoder, buf, bufLen);
@ -2667,7 +2694,7 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
  if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1;
  if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1;
  if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->cacheLast) < 0) return -1;
  if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1;
  for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) {
    SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i);
@ -2706,7 +2733,7 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) {
  if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1;
  if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1;
  if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->cacheLast) < 0) return -1;
  if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1;
  if (pRsp->numOfRetensions > 0) {
    pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention));
@ -3617,7 +3644,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -3632,7 +3659,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
  if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->standby) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
@ -3675,7 +3702,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -3690,7 +3717,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
  if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->standby) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
@ -3800,7 +3827,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -3808,7 +3835,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq
  if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
  for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
@ -3831,7 +3858,7 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -3839,7 +3866,7 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
  if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
  for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@ -210,7 +210,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
    return -1;
  }

-  dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d", createReq.vgId, createReq.isTsma, createReq.standby);
+  dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d", createReq.vgId,
+         createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize);
  vmGenerateVnodeCfg(&createReq, &vnodeCfg);

  if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) {
--- a/source/dnode/mgmt/test/vnode/vnode.cpp
+++ b/source/dnode/mgmt/test/vnode/vnode.cpp
@ -45,7 +45,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
    createReq.compression = 2;
    createReq.replica = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.selfIndex = 0;
    for (int r = 0; r < createReq.replica; ++r) {
      SReplica* pReplica = &createReq.replicas[r];
@ -80,7 +80,7 @@ TEST_F(DndTestVnode, 02_Alter_Vnode) {
    alterReq.walLevel = 1;
    alterReq.replica = 1;
    alterReq.strict = 1;
-    alterReq.cacheLastRow = 0;
+    alterReq.cacheLast = 0;
    alterReq.selfIndex = 0;
    for (int r = 0; r < alterReq.replica; ++r) {
      SReplica* pReplica = &alterReq.replicas[r];
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@ -246,7 +246,7 @@ typedef struct {
  int32_t buffer;
  int32_t pageSize;
  int32_t pages;
-  int32_t lastRowMem;
+  int32_t cacheLastSize;
  int32_t daysPerFile;
  int32_t daysToKeep0;
  int32_t daysToKeep1;
@ -260,7 +260,7 @@ typedef struct {
  int8_t  replications;
  int8_t  strict;
  int8_t  hashMethod;  // default is 1
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int32_t numOfRetensions;
  SArray* pRetensions;
  int8_t  schemaless;
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@ -15,11 +15,11 @@

 #define _DEFAULT_SOURCE
 #include "mndConsumer.h"
-#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
 #include "mndOffset.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndStb.h"
 #include "mndSubscribe.h"
@ -435,17 +435,6 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
      goto SUBSCRIBE_OVER;
    }

-#if 0
-    // ref topic to prevent drop
-    // TODO make topic complete
-    SMqTopicObj topicObj = {0};
-    memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
-    topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1;
-    mInfo("subscribe topic %s by consumer:%" PRId64 ",cgroup %s, refcnt %d", pTopic->name, consumerId, cgroup,
-          topicObj.refConsumerCnt);
-    if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
-#endif
-
    mndReleaseTopic(pMnode, pTopic);
  }

@ -472,8 +461,8 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {

    int32_t status = atomic_load_32(&pConsumerOld->status);

-    mInfo("receive subscribe request from old consumer:%" PRId64 ", current status: %s", consumerId,
-          mndConsumerStatusName(status));
+    mInfo("receive subscribe request from existing consumer:%" PRId64 ", current status: %s, subscribe topic num: %d",
+          consumerId, mndConsumerStatusName(status), newTopicNum);

    if (status != MQ_CONSUMER_STATUS__READY) {
      terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
@ -849,12 +838,15 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
    pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
    if (pShow->pIter == NULL) break;
    if (taosArrayGetSize(pConsumer->assignedTopics) == 0) {
+      mDebug("showing consumer %ld no assigned topic, skip", pConsumer->consumerId);
      sdbRelease(pSdb, pConsumer);
      continue;
    }

    taosRLockLatch(&pConsumer->lock);

+    mDebug("showing consumer %ld", pConsumer->consumerId);
+
    int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics);
    bool    hasTopic = true;
    if (topicSz == 0) {
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@ -93,7 +93,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.buffer, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pageSize, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pages, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.lastRowMem, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheLastSize, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, _OVER)
@ -106,7 +106,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER)
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, _OVER)
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.strict, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLast, _OVER)
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER)
  for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) {
@ -166,7 +166,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.buffer, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pageSize, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pages, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.lastRowMem, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheLastSize, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, _OVER)
@ -179,7 +179,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER)
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, _OVER)
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.strict, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLast, _OVER)
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.hashMethod, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, _OVER)
  if (pDb->cfg.numOfRetensions > 0) {
@ -234,7 +234,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
  pOld->cfg.buffer = pNew->cfg.buffer;
  pOld->cfg.pageSize = pNew->cfg.pageSize;
  pOld->cfg.pages = pNew->cfg.pages;
-  pOld->cfg.lastRowMem = pNew->cfg.lastRowMem;
+  pOld->cfg.cacheLastSize = pNew->cfg.cacheLastSize;
  pOld->cfg.daysPerFile = pNew->cfg.daysPerFile;
  pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0;
  pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1;
@ -242,7 +242,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
  pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod;
  pOld->cfg.walLevel = pNew->cfg.walLevel;
  pOld->cfg.strict = pNew->cfg.strict;
-  pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow;
+  pOld->cfg.cacheLast = pNew->cfg.cacheLast;
  pOld->cfg.replications = pNew->cfg.replications;
  taosWUnLockLatch(&pOld->lock);
  return 0;
@ -291,7 +291,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
  if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
  if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1;
  if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1;
-  if (pCfg->lastRowMem < TSDB_MIN_DB_LAST_ROW_MEM || pCfg->lastRowMem > TSDB_MAX_DB_LAST_ROW_MEM) return -1;
+  if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_LAST_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_LAST_SIZE) return -1;
  if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
  if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
  if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
@ -310,7 +310,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
  if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
  if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
  if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1;
-  if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
+  if (pCfg->cacheLast < TSDB_MIN_DB_CACHE_LAST || pCfg->cacheLast > TSDB_MAX_DB_CACHE_LAST) return -1;
  if (pCfg->hashMethod != 1) return -1;
  if (pCfg->replications > mndGetDnodeSize(pMnode)) {
    terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
@ -339,8 +339,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
  if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;
  if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA;
  if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT;
-  if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
-  if (pCfg->lastRowMem <= 0) pCfg->lastRowMem = TSDB_DEFAULT_LAST_ROW_MEM;
+  if (pCfg->cacheLast < 0) pCfg->cacheLast = TSDB_DEFAULT_CACHE_LAST;
+  if (pCfg->cacheLastSize <= 0) pCfg->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE;
  if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
  if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF;
 }
@ -439,7 +439,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
      .buffer = pCreate->buffer,
      .pageSize = pCreate->pageSize,
      .pages = pCreate->pages,
-      .lastRowMem = pCreate->lastRowMem,
+      .cacheLastSize = pCreate->cacheLastSize,
      .daysPerFile = pCreate->daysPerFile,
      .daysToKeep0 = pCreate->daysToKeep0,
      .daysToKeep1 = pCreate->daysToKeep1,
@ -452,7 +452,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
      .compression = pCreate->compression,
      .replications = pCreate->replications,
      .strict = pCreate->strict,
-      .cacheLastRow = pCreate->cacheLastRow,
+      .cacheLast = pCreate->cacheLast,
      .hashMethod = 1,
      .schemaless = pCreate->schemaless,
  };
@ -623,13 +623,13 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
 #endif
  }

-  if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) {
-    pDb->cfg.cacheLastRow = pAlter->cacheLastRow;
+  if (pAlter->cacheLast >= 0 && pAlter->cacheLast != pDb->cfg.cacheLast) {
+    pDb->cfg.cacheLast = pAlter->cacheLast;
    terrno = 0;
  }

-  if (pAlter->lastRowMem > 0 && pAlter->lastRowMem != pDb->cfg.lastRowMem) {
-    pDb->cfg.lastRowMem = pAlter->lastRowMem;
+  if (pAlter->cacheLastSize > 0 && pAlter->cacheLastSize != pDb->cfg.cacheLastSize) {
+    pDb->cfg.cacheLastSize = pAlter->cacheLastSize;
    terrno = 0;
  }

@ -801,7 +801,7 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
  cfgRsp.compression = pDb->cfg.compression;
  cfgRsp.replications = pDb->cfg.replications;
  cfgRsp.strict = pDb->cfg.strict;
-  cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow;
+  cfgRsp.cacheLast = pDb->cfg.cacheLast;
  cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
  cfgRsp.pRetensions = pDb->cfg.pRetensions;
  cfgRsp.schemaless = pDb->cfg.schemaless;
@ -1467,7 +1467,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false);

    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLast, false);

    const char *precStr = NULL;
    switch (pDb->cfg.precision) {
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@ -527,10 +527,20 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
  streamObj.version = 1;
  streamObj.sql = pCreate->sql;
  streamObj.smaId = smaObj.uid;
-  streamObj.watermark = 0;
-  streamObj.trigger = STREAM_TRIGGER_AT_ONCE;
+  streamObj.watermark = pCreate->watermark;
+  streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE;
+  streamObj.triggerParam = pCreate->maxDelay;
  streamObj.ast = strdup(smaObj.ast);

+  // check the maxDelay
+  if (streamObj.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) {
+    int64_t msInterval = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND);
+    streamObj.triggerParam = msInterval > TSDB_MIN_ROLLUP_MAX_DELAY ? msInterval : TSDB_MIN_ROLLUP_MAX_DELAY;
+  }
+  if (streamObj.triggerParam > TSDB_MAX_ROLLUP_MAX_DELAY) {
+    streamObj.triggerParam = TSDB_MAX_ROLLUP_MAX_DELAY;
+  }
+
  if (mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg) != 0) {
    mError("sma:%s, failed to create since %s", smaObj.name, terrstr());
    return -1;
@ -1131,14 +1141,17 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
  SSmaObj *pSma = NULL;
  int32_t  cols = 0;

-  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
-  if (pDb == NULL) return 0;
+  SDbObj *pDb = NULL;
+  if (strlen(pShow->db) > 0) {
+    pDb = mndAcquireDb(pMnode, pShow->db);
+    if (pDb == NULL) return 0;
+  }

  while (numOfRows < rows) {
    pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma);
    if (pShow->pIter == NULL) break;

-    if (pSma->dbUid != pDb->uid) {
+    if (NULL != pDb && pSma->dbUid != pDb->uid) {
      sdbRelease(pSdb, pSma);
      continue;
    }
@ -1151,7 +1164,7 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
    STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName));

    char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
-    STR_TO_VARSTR(n2, (char *)mndGetDbStr(pDb->name));
+    STR_TO_VARSTR(n2, (char *)mndGetDbStr(pSma->db));

    SName stbName = {0};
    tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@ -207,7 +207,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
  createReq.buffer = pDb->cfg.buffer;
  createReq.pageSize = pDb->cfg.pageSize;
  createReq.pages = pDb->cfg.pages;
-  createReq.lastRowMem = pDb->cfg.lastRowMem;
+  createReq.cacheLastSize = pDb->cfg.cacheLastSize;
  createReq.daysPerFile = pDb->cfg.daysPerFile;
  createReq.daysToKeep0 = pDb->cfg.daysToKeep0;
  createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
@ -219,7 +219,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
  createReq.precision = pDb->cfg.precision;
  createReq.compression = pDb->cfg.compression;
  createReq.strict = pDb->cfg.strict;
-  createReq.cacheLastRow = pDb->cfg.cacheLastRow;
+  createReq.cacheLast = pDb->cfg.cacheLast;
  createReq.replica = pVgroup->replica;
  createReq.selfIndex = -1;
  createReq.hashBegin = pVgroup->hashBegin;
@ -277,7 +277,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_
  alterReq.buffer = pDb->cfg.buffer;
  alterReq.pageSize = pDb->cfg.pageSize;
  alterReq.pages = pDb->cfg.pages;
-  alterReq.lastRowMem = pDb->cfg.lastRowMem;
+  alterReq.cacheLastSize = pDb->cfg.cacheLastSize;
  alterReq.daysPerFile = pDb->cfg.daysPerFile;
  alterReq.daysToKeep0 = pDb->cfg.daysToKeep0;
  alterReq.daysToKeep1 = pDb->cfg.daysToKeep1;
@ -285,7 +285,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_
  alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod;
  alterReq.walLevel = pDb->cfg.walLevel;
  alterReq.strict = pDb->cfg.strict;
-  alterReq.cacheLastRow = pDb->cfg.cacheLastRow;
+  alterReq.cacheLast = pDb->cfg.cacheLast;
  alterReq.replica = pVgroup->replica;

  for (int32_t v = 0; v < pVgroup->replica; ++v) {
@ -742,8 +742,8 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDbInput, SVgObj *pVgroup) {
  int64_t vgroupMemroy = 0;
  if (pDb != NULL) {
    vgroupMemroy = (int64_t)pDb->cfg.buffer * 1024 * 1024 + (int64_t)pDb->cfg.pages * pDb->cfg.pageSize * 1024;
-    if (pDb->cfg.cacheLastRow > 0) {
-      vgroupMemroy += (int64_t)pDb->cfg.lastRowMem * 1024 * 1024;
+    if (pDb->cfg.cacheLast > 0) {
+      vgroupMemroy += (int64_t)pDb->cfg.cacheLastSize * 1024 * 1024;
    }
  }

--- a/source/dnode/mnode/impl/test/db/db.cpp
+++ b/source/dnode/mnode/impl/test/db/db.cpp
@ -50,7 +50,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;
    createReq.numOfStables = 0;
    createReq.numOfRetensions = 0;
@ -84,7 +84,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
    alterdbReq.fsyncPeriod = 4000;
    alterdbReq.walLevel = 2;
    alterdbReq.strict = 1;
-    alterdbReq.cacheLastRow = 1;
+    alterdbReq.cacheLast = 1;
    alterdbReq.replications = 1;

    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
@ -146,7 +146,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;
    createReq.numOfStables = 0;
    createReq.numOfRetensions = 0;
--- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp
+++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp
@ -288,7 +288,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;
    createReq.numOfStables = 0;
    createReq.numOfRetensions = 0;
@ -319,7 +319,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) {
    alterdbReq.fsyncPeriod = 4000;
    alterdbReq.walLevel = 2;
    alterdbReq.strict = 1;
-    alterdbReq.cacheLastRow = 1;
+    alterdbReq.cacheLast = 1;
    alterdbReq.replications = 3;

    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
@ -345,7 +345,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) {
    alterdbReq.fsyncPeriod = 4000;
    alterdbReq.walLevel = 2;
    alterdbReq.strict = 1;
-    alterdbReq.cacheLastRow = 1;
+    alterdbReq.cacheLast = 1;
    alterdbReq.replications = 1;

    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
--- a/source/dnode/mnode/impl/test/sma/sma.cpp
+++ b/source/dnode/mnode/impl/test/sma/sma.cpp
@ -55,7 +55,7 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
  createReq.compression = 2;
  createReq.replications = 1;
  createReq.strict = 1;
-  createReq.cacheLastRow = 0;
+  createReq.cacheLast = 0;
  createReq.ignoreExist = 1;

  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/dnode/mnode/impl/test/stb/stb.cpp
+++ b/source/dnode/mnode/impl/test/stb/stb.cpp
@ -56,7 +56,7 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
  createReq.compression = 2;
  createReq.replications = 1;
  createReq.strict = 1;
-  createReq.cacheLastRow = 0;
+  createReq.cacheLast = 0;
  createReq.ignoreExist = 1;

  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/dnode/mnode/impl/test/topic/topic.cpp
+++ b/source/dnode/mnode/impl/test/topic/topic.cpp
@ -48,7 +48,7 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
  createReq.compression = 2;
  createReq.replications = 1;
  createReq.strict = 1;
-  createReq.cacheLastRow = 0;
+  createReq.cacheLast = 0;
  createReq.ignoreExist = 1;

  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/dnode/mnode/impl/test/user/user.cpp
+++ b/source/dnode/mnode/impl/test/user/user.cpp
@ -315,7 +315,7 @@ TEST_F(MndTestUser, 03_Alter_User) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;

    int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@ -3,6 +3,7 @@ add_library(vnode STATIC "")
 target_sources(
    vnode
    PRIVATE
+
    # vnode
    "src/vnd/vnodeOpen.c"
    "src/vnd/vnodeBufPool.c"
@ -13,7 +14,6 @@ target_sources(
    "src/vnd/vnodeSvr.c"
    "src/vnd/vnodeSync.c"
    "src/vnd/vnodeSnapshot.c"
-    "src/vnd/vnodeUtil.c"

    # meta
    "src/meta/metaOpen.c"
@ -46,6 +46,7 @@ target_sources(
    "src/tsdb/tsdbUtil.c"
    "src/tsdb/tsdbSnapshot.c"
    "src/tsdb/tsdbCacheRead.c"
+    "src/tsdb/tsdbRetention.c"

    # tq
    "src/tq/tq.c"
@ -62,7 +63,6 @@ target_include_directories(
    PUBLIC "inc"
    PRIVATE "src/inc"
    PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
-    
 )
 target_link_libraries(
    vnode
@ -76,18 +76,19 @@ target_link_libraries(
    PUBLIC executor
    PUBLIC scheduler
    PUBLIC tdb
-    #PUBLIC bdb
-    #PUBLIC scalar
+
+    # PUBLIC bdb
+    # PUBLIC scalar
    PUBLIC transport
    PUBLIC stream
    PUBLIC index
 )
 target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
-if (${BUILD_WITH_INVERTEDINDEX})
-  add_definitions(-DUSE_INVERTED_INDEX) 
+
+if(${BUILD_WITH_INVERTEDINDEX})
+    add_definitions(-DUSE_INVERTED_INDEX)
 endif(${BUILD_WITH_INVERTEDINDEX})
+
 if(${BUILD_TEST})
    add_subdirectory(test)
 endif(${BUILD_TEST})
-
-
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@ -38,10 +38,11 @@ extern "C" {
 #endif

 // vnode
-typedef struct SVnode           SVnode;
-typedef struct STsdbCfg         STsdbCfg;  // todo: remove
-typedef struct SVnodeCfg        SVnodeCfg;
-typedef struct SVSnapshotReader SVSnapshotReader;
+typedef struct SVnode       SVnode;
+typedef struct STsdbCfg     STsdbCfg;  // todo: remove
+typedef struct SVnodeCfg    SVnodeCfg;
+typedef struct SVSnapReader SVSnapReader;
+typedef struct SVSnapWriter SVSnapWriter;

 extern const SVnodeCfg vnodeCfgDefault;

@ -57,10 +58,6 @@ void    vnodeStop(SVnode *pVnode);
 int64_t vnodeGetSyncHandle(SVnode *pVnode);
 void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
 void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever);
-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
-
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
 int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
@ -133,7 +130,7 @@ bool    tsdbNextDataBlock(STsdbReader *pReader);
 void    tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
 int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
 SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
-int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx);
+int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
 int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
 int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
 void   *tsdbGetIdx(SMeta *pMeta);
@ -185,7 +182,14 @@ int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader);
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

-// need to reposition
+// SVSnapReader
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader);
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData);
+// SVSnapWriter
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback);
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);

 // structs
 struct STsdbCfg {
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@ -57,6 +57,9 @@ int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
 // metaCommit ==================
 static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }

+// metaTable ==================
+int metaHandleEntry(SMeta* pMeta, const SMetaEntry* pME);
+
 struct SMeta {
  TdThreadRwlock lock;

--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@ -49,6 +49,7 @@ struct SSmaEnv {
 typedef struct {
  int8_t  inited;
  int32_t rsetId;
+  void   *tmrHandle;  // shared by all fetch tasks
 } SSmaMgmt;

 #define SMA_ENV_LOCK(env) ((env)->lock)
@ -65,7 +66,6 @@ struct SRSmaStat {
  SSma     *pSma;
  int64_t   submitVer;
  int64_t   refId;         // shared by fetch tasks
-  void     *tmrHandle;     // shared by fetch tasks
  int8_t    triggerStat;   // shared by fetch tasks
  int8_t    runningStat;   // for persistence task 
  SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
@ -82,7 +82,6 @@ struct SSmaStat {
 #define SMA_TSMA_STAT(s)     (&(s)->tsmaStat)
 #define SMA_RSMA_STAT(s)     (&(s)->rsmaStat)
 #define RSMA_INFO_HASH(r)    ((r)->rsmaInfoHash)
-#define RSMA_TMR_HANDLE(r)   ((r)->tmrHandle)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
 #define RSMA_REF_ID(r)       ((r)->refId)
@ -189,7 +188,7 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {

 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
-void          *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat);

 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@ -89,6 +89,8 @@ typedef struct {
    STqExecTb  execTb;
    STqExecDb  execDb;
  };
+  // TODO remove it
+  int64_t tsdbEndVer;

 } STqExecHandle;

@ -129,7 +131,7 @@ typedef struct {
 static STqMgmt tqMgmt = {0};

 // tqRead
-int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset);
+int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset);
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);

 // tqExec
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@ -64,6 +64,7 @@ typedef struct SRowIter       SRowIter;
 typedef struct STsdbFS        STsdbFS;
 typedef struct SRowMerger     SRowMerger;
 typedef struct STsdbFSState   STsdbFSState;
+typedef struct STsdbSnapHdr   STsdbSnapHdr;

 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
@ -237,7 +238,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf);
 // tsdbCache
 int32_t tsdbOpenCache(STsdb *pTsdb);
 void    tsdbCloseCache(SLRUCache *pCache);
-int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row);
+int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
 int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup);
 int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
 int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@ -49,17 +49,20 @@
 extern "C" {
 #endif

-typedef struct SVnodeInfo          SVnodeInfo;
-typedef struct SMeta               SMeta;
-typedef struct SSma                SSma;
-typedef struct STsdb               STsdb;
-typedef struct STQ                 STQ;
-typedef struct SVState             SVState;
-typedef struct SVBufPool           SVBufPool;
-typedef struct SQWorker            SQHandle;
-typedef struct STsdbKeepCfg        STsdbKeepCfg;
-typedef struct SMetaSnapshotReader SMetaSnapshotReader;
-typedef struct STsdbSnapshotReader STsdbSnapshotReader;
+typedef struct SVnodeInfo      SVnodeInfo;
+typedef struct SMeta           SMeta;
+typedef struct SSma            SSma;
+typedef struct STsdb           STsdb;
+typedef struct STQ             STQ;
+typedef struct SVState         SVState;
+typedef struct SVBufPool       SVBufPool;
+typedef struct SQWorker        SQHandle;
+typedef struct STsdbKeepCfg    STsdbKeepCfg;
+typedef struct SMetaSnapReader SMetaSnapReader;
+typedef struct SMetaSnapWriter SMetaSnapWriter;
+typedef struct STsdbSnapReader STsdbSnapReader;
+typedef struct STsdbSnapWriter STsdbSnapWriter;
+typedef struct SSnapDataHdr    SSnapDataHdr;

 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@ -72,10 +75,8 @@ typedef struct STsdbSnapshotReader STsdbSnapshotReader;
 #define VNODE_RSMA2_DIR "rsma2"

 // vnd.h
-void*   vnodeBufPoolMalloc(SVBufPool* pPool, int size);
-void    vnodeBufPoolFree(SVBufPool* pPool, void* p);
-int32_t vnodeRealloc(void** pp, int32_t size);
-void    vnodeFree(void* p);
+void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
+void  vnodeBufPoolFree(SVBufPool* pPool, void* p);

 // meta
 typedef struct SMCtbCursor SMCtbCursor;
@ -109,9 +110,6 @@ STSma*          metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
 STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
 SArray*         metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*         metaGetSmaTbUids(SMeta* pMeta);
-int32_t         metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t         metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
-int32_t         metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
 void*           metaGetIvtIdx(SMeta* pMeta);
 int             metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
@ -131,9 +129,6 @@ int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* p
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                                void* pMemRef);
-int32_t     tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t     tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader);
-int32_t     tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData);

 // tq
 int     tqInit();
@ -183,6 +178,23 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
 void    tdUidStoreDestory(STbUidStore* pStore);
 void*   tdUidStoreFree(STbUidStore* pStore);

+// SMetaSnapReader ========================================
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
+// SMetaSnapWriter ========================================
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback);
+// STsdbSnapReader ========================================
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader);
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData);
+// STsdbSnapWriter ========================================
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter);
+int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
+
 typedef struct {
  int8_t  streamType;  // sma or other
  int8_t  dstType;
@ -202,7 +214,9 @@ typedef struct {
 struct SVState {
  int64_t committed;
  int64_t applied;
+  int64_t applyTerm;
  int64_t commitID;
+  int64_t commitTerm;
 };

 struct SVnodeInfo {
@ -291,6 +305,12 @@ struct SSma {
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);

+struct SSnapDataHdr {
+  int8_t  type;
+  int64_t size;
+  uint8_t data[];
+};
+
 #ifdef __cplusplus
 }
 #endif
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@ -147,6 +147,8 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
      return -1;
    }

+    tDecoderClear(&pTbCur->mr.coder);
+
    metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey);
    if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) {
      continue;
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@ -15,53 +15,57 @@

 #include "meta.h"

-struct SMetaSnapshotReader {
+// SMetaSnapReader ========================================
+struct SMetaSnapReader {
  SMeta*  pMeta;
-  TBC*    pTbc;
  int64_t sver;
  int64_t ever;
+  TBC*    pTbc;
 };

-int32_t metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  int32_t              code = 0;
-  int32_t              c = 0;
-  SMetaSnapshotReader* pMetaReader = NULL;
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
+  int32_t          code = 0;
+  int32_t          c = 0;
+  SMetaSnapReader* pMetaSnapReader = NULL;

-  pMetaReader = (SMetaSnapshotReader*)taosMemoryCalloc(1, sizeof(*pMetaReader));
-  if (pMetaReader == NULL) {
+  // alloc
+  pMetaSnapReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaSnapReader));
+  if (pMetaSnapReader == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
  }
-  pMetaReader->pMeta = pMeta;
-  pMetaReader->sver = sver;
-  pMetaReader->ever = ever;
-  code = tdbTbcOpen(pMeta->pTbDb, &pMetaReader->pTbc, NULL);
+  pMetaSnapReader->pMeta = pMeta;
+  pMetaSnapReader->sver = sver;
+  pMetaSnapReader->ever = ever;
+
+  // impl
+  code = tdbTbcOpen(pMeta->pTbDb, &pMetaSnapReader->pTbc, NULL);
  if (code) {
    goto _err;
  }

-  code = tdbTbcMoveTo(pMetaReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
+  code = tdbTbcMoveTo(pMetaSnapReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
  if (code) {
    goto _err;
  }

-  *ppReader = pMetaReader;
+  *ppReader = pMetaSnapReader;
  return code;

 _err:
+  metaError("vgId:%d meta snap reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
  *ppReader = NULL;
  return code;
 }

-int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader) {
-  if (pReader) {
-    tdbTbcClose(pReader->pTbc);
-    taosMemoryFree(pReader);
-  }
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
+  tdbTbcClose((*ppReader)->pTbc);
+  taosMemoryFree(*ppReader);
+  *ppReader = NULL;
  return 0;
 }

-int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nDatap) {
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
  const void* pKey = NULL;
  const void* pData = NULL;
  int32_t     nKey = 0;
@ -71,23 +75,110 @@ int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t*
  for (;;) {
    code = tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData);
    if (code || ((STbDbKey*)pData)->version > pReader->ever) {
-      return TSDB_CODE_VND_READ_END;
+      code = TSDB_CODE_VND_READ_END;
+      goto _exit;
    }

    if (((STbDbKey*)pData)->version < pReader->sver) {
+      tdbTbcMoveToNext(pReader->pTbc);
      continue;
    }

+    tdbTbcMoveToNext(pReader->pTbc);
    break;
  }

  // copy the data
-  if (vnodeRealloc(ppData, nData) < 0) {
+  if (tRealloc(ppData, sizeof(SSnapDataHdr) + nData) < 0) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    return code;
  }
+  ((SSnapDataHdr*)(*ppData))->type = 0;  // TODO: use macro
+  ((SSnapDataHdr*)(*ppData))->size = nData;
+  memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData);

-  memcpy(*ppData, pData, nData);
-  *nDatap = nData;
+_exit:
+  return code;
+}
+
+// SMetaSnapWriter ========================================
+struct SMetaSnapWriter {
+  SMeta*  pMeta;
+  int64_t sver;
+  int64_t ever;
+};
+
+static int32_t metaSnapRollback(SMetaSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t metaSnapCommit(SMetaSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter;
+
+  // alloc
+  pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pMeta = pMeta;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  *ppWriter = pWriter;
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  *ppWriter = NULL;
+  return code;
+}
+
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter = *ppWriter;
+
+  if (rollback) {
+    code = metaSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = metaSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+  taosMemoryFree(pWriter);
+  *ppWriter = NULL;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer close failed since %s", TD_VID(pWriter->pMeta->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t    code = 0;
+  SMeta*     pMeta = pWriter->pMeta;
+  SMetaEntry metaEntry = {0};
+  SDecoder*  pDecoder = &(SDecoder){0};
+
+  tDecoderInit(pDecoder, pData, nData);
+  metaDecodeEntry(pDecoder, &metaEntry);
+
+  code = metaHandleEntry(pMeta, &metaEntry);
+  if (code) goto _err;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
  return code;
 }
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@ -17,7 +17,6 @@

 static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
 static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
@ -51,7 +50,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
  if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
    return -1;
  }
-  void *      data = pCtbEntry->ctbEntry.pTags;
+  void       *data = pCtbEntry->ctbEntry.pTags;
  const char *tagName = pSchema->name;

  tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
@ -70,7 +69,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
    char     type = pTagVal->type;

-    char *  key = pTagVal->pKey;
+    char   *key = pTagVal->pKey;
    int32_t nKey = strlen(key);

    SIndexTerm *term = NULL;
@ -78,7 +77,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
      term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
    } else if (type == TSDB_DATA_TYPE_NCHAR) {
      if (pTagVal->nData > 0) {
-        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        char   *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
        int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
        memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
        type = TSDB_DATA_TYPE_VARCHAR;
@ -109,7 +108,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
  if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
    return -1;
  }
-  void *      data = pCtbEntry->ctbEntry.pTags;
+  void       *data = pCtbEntry->ctbEntry.pTags;
  const char *tagName = pSchema->name;

  tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
@ -128,7 +127,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
    char     type = pTagVal->type;

-    char *  key = pTagVal->pKey;
+    char   *key = pTagVal->pKey;
    int32_t nKey = strlen(key);

    SIndexTerm *term = NULL;
@ -136,7 +135,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
      term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
    } else if (type == TSDB_DATA_TYPE_NCHAR) {
      if (pTagVal->nData > 0) {
-        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        char   *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
        int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
        memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
        type = TSDB_DATA_TYPE_VARCHAR;
@ -169,9 +168,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
  int         vLen = 0;
  const void *pKey = NULL;
  const void *pVal = NULL;
-  void *      pBuf = NULL;
+  void       *pBuf = NULL;
  int32_t     szBuf = 0;
-  void *      p = NULL;
+  void       *p = NULL;
  SMetaReader mr = {0};

  // validate req
@ -229,7 +228,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
  }

  // drop all child tables
-  TBC *   pCtbIdxc = NULL;
+  TBC    *pCtbIdxc = NULL;
  SArray *pArray = taosArrayInit(8, sizeof(tb_uid_t));

  tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
@ -285,8 +284,8 @@ _exit:
 int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
  SMetaEntry  oStbEntry = {0};
  SMetaEntry  nStbEntry = {0};
-  TBC *       pUidIdxc = NULL;
-  TBC *       pTbDbc = NULL;
+  TBC        *pUidIdxc = NULL;
+  TBC        *pTbDbc = NULL;
  const void *pData;
  int         nData;
  int64_t     oversion;
@ -409,7 +408,7 @@ _err:
 }

 int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
-  void *   pData = NULL;
+  void    *pData = NULL;
  int      nData = 0;
  int      rc = 0;
  tb_uid_t uid;
@ -477,7 +476,7 @@ static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
 }

 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
-  void *     pData = NULL;
+  void      *pData = NULL;
  int        nData = 0;
  int        rc = 0;
  SMetaEntry e = {0};
@ -538,14 +537,14 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
 }

 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
-  void *          pVal = NULL;
+  void           *pVal = NULL;
  int             nVal = 0;
-  const void *    pData = NULL;
+  const void     *pData = NULL;
  int             nData = 0;
  int             ret = 0;
  tb_uid_t        uid;
  int64_t         oversion;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
  SMetaEntry      entry = {0};
  SSchemaWrapper *pSchema;
  int             c;
@ -699,7 +698,7 @@ _err:
 static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
  SMetaEntry  ctbEntry = {0};
  SMetaEntry  stbEntry = {0};
-  void *      pVal = NULL;
+  void       *pVal = NULL;
  int         nVal = 0;
  int         ret;
  int         c;
@ -730,7 +729,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
  oversion = *(int64_t *)pData;

  // search table.db
-  TBC *    pTbDbc = NULL;
+  TBC     *pTbDbc = NULL;
  SDecoder dc1 = {0};
  SDecoder dc2 = {0};

@ -754,7 +753,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
  metaDecodeEntry(&dc2, &stbEntry);

  SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
  int32_t         iCol = 0;
  for (;;) {
    pColumn = NULL;
@ -784,8 +783,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
    memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
  } else {
    const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags;
-    STag *      pNewTag = NULL;
-    SArray *    pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
+    STag       *pNewTag = NULL;
+    SArray     *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
    if (!pTagArray) {
      terrno = TSDB_CODE_OUT_OF_MEMORY;
      goto _err;
@ -844,7 +843,7 @@ _err:
 }

 static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
-  void *      pVal = NULL;
+  void       *pVal = NULL;
  int         nVal = 0;
  const void *pData = NULL;
  int         nData = 0;
@ -948,8 +947,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta

 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
  STbDbKey tbDbKey;
-  void *   pKey = NULL;
-  void *   pVal = NULL;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
  int      kLen = 0;
  int      vLen = 0;
  SEncoder coder = {0};
@ -1055,14 +1054,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
 }

 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
-  void *         pData = NULL;
+  void          *pData = NULL;
  int            nData = 0;
  STbDbKey       tbDbKey = {0};
  SMetaEntry     stbEntry = {0};
-  STagIdxKey *   pTagIdxKey = NULL;
+  STagIdxKey    *pTagIdxKey = NULL;
  int32_t        nTagIdxKey;
  const SSchema *pTagColumn;       // = &stbEntry.stbEntry.schema.pSchema[0];
-  const void *   pTagData = NULL;  //
+  const void    *pTagData = NULL;  //
  int32_t        nTagData = 0;
  SDecoder       dc = {0};

@ -1109,7 +1108,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {

 static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
  SEncoder              coder = {0};
-  void *                pVal = NULL;
+  void                 *pVal = NULL;
  int                   vLen = 0;
  int                   rcode = 0;
  SSkmDbKey             skmDbKey = {0};
@ -1151,7 +1150,7 @@ _exit:
  return rcode;
 }

-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
+int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
  metaWLock(pMeta);

  // save to table.db
--- a/source/dnode/vnode/src/sma/smaEnv.c
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@ -49,16 +49,26 @@ int32_t smaInit() {
  }

  if (old == 0) {
+    // init tref rset
    smaMgmt.rsetId = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);

    if (smaMgmt.rsetId < 0) {
-      smaError("failed to init sma rset since %s", terrstr());
      atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma rset since %s", terrstr());
+      return TSDB_CODE_FAILED;
+    }
+
+    // init fetch timer handle
+    smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");
+    if (!smaMgmt.tmrHandle) {
+      taosCloseRef(smaMgmt.rsetId);
+      atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma tmr hanle since %s", terrstr());
      return TSDB_CODE_FAILED;
    }

-    smaInfo("sma rset is initialized, rsetId:%d", smaMgmt.rsetId);
    atomic_store_8(&smaMgmt.inited, 1);
+    smaInfo("sma mgmt env is initialized, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
  }

  return TSDB_CODE_SUCCESS;
@ -81,8 +91,9 @@ void smaCleanUp() {
  }

  if (old == 1) {
-    smaInfo("sma rset is cleaned up, resetId:%d", smaMgmt.rsetId);
    taosCloseRef(smaMgmt.rsetId);
+    taosTmrCleanUp(smaMgmt.tmrHandle);
+    smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
    atomic_store_8(&smaMgmt.inited, 0);
  }
 }
@ -203,20 +214,11 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
      }
      pRSmaStat->refId = refId;

-      // init timer
-      RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA");
-      if (!RSMA_TMR_HANDLE(pRSmaStat)) {
-        taosMemoryFreeClear(*pSmaStat);
-        return TSDB_CODE_FAILED;
-      }

      // init hash
      RSMA_INFO_HASH(pRSmaStat) = taosHashInit(
          RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
      if (!RSMA_INFO_HASH(pRSmaStat)) {
-        if (RSMA_TMR_HANDLE(pRSmaStat)) {
-          taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat));
-        }
        taosMemoryFreeClear(*pSmaStat);
        return TSDB_CODE_FAILED;
      }
@ -277,7 +279,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
      void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
      while (infoHash) {
        SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
-        tdFreeRSmaInfo(pSmaInfo);
+        tdFreeRSmaInfo(pSma, pSmaInfo);
        infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
      }
    }
@ -298,11 +300,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
        nLoops = 0;
      }
    }
-
-    // step 6: cleanup the timer handle
-    if (RSMA_TMR_HANDLE(pStat)) {
-      taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
-    }
  }
 }

--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@ -27,16 +27,19 @@ SSmaMgmt smaMgmt = {
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;

-static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
-static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle,
-                                       int8_t idx);
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
-                                 tb_uid_t suid, int8_t level);
-static void    tdRSmaFetchTrigger(void *param, void *tmrId);
-static void    tdRSmaPersistTrigger(void *param, void *tmrId);
-static void   *tdRSmaPersistExec(void *param);
-static void    tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
+static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static int32_t    tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                          SReadHandle *handle, int8_t idx);
+static int32_t    tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
+                                    STSchema *pTSchema, tb_uid_t suid, int8_t level);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid);
+static int32_t    tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                             int8_t blkType);
+static void       tdRSmaFetchTrigger(void *param, void *tmrId);
+static void       tdRSmaPersistTrigger(void *param, void *tmrId);
+static void      *tdRSmaPersistExec(void *param);
+static void       tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);

 static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
@ -48,25 +51,26 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);

 struct SRSmaInfoItem {
-  SRSmaInfo *pRsmaInfo;
-  int64_t    refId;
-  void      *taskInfo;  // qTaskInfo_t
-  tmr_h      tmrId;
-  int8_t     level;
-  int8_t     tmrInitFlag;
-  int8_t     triggerStat;
-  int32_t    maxDelay;
+  void   *taskInfo;  // qTaskInfo_t
+  int64_t refId;
+  tmr_h   tmrId;
+  int32_t maxDelay;
+  int8_t  level;
+  int8_t  triggerStat;
 };

 struct SRSmaInfo {
  STSchema     *pTSchema;
-  SRSmaStat    *pStat;
  int64_t       suid;
  SRSmaInfoItem items[TSDB_RETENTION_L2];
 };

-#define RSMA_INFO_SMA(r)  ((r)->pStat->pSma)
-#define RSMA_INFO_STAT(r) ((r)->pStat)
+static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
+  // adapt accordingly if definition of SRSmaInfo update
+  int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
+  ASSERT(pItem->level == 1 || pItem->level == 2);
+  return (SRSmaInfo *)POINTER_SHIFT(pItem, -sizeof(SRSmaInfoItem) * (pItem->level - 1) - rsmaInfoHeadLen);
+}

 struct SRSmaQTaskInfoItem {
  int32_t len;
@ -108,9 +112,8 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId,
  }
 }

-void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
+void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
  if (pInfo) {
-    SSma *pSma = RSMA_INFO_SMA(pInfo);
    for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
      SRSmaInfoItem *pItem = &pInfo->items[i];
      if (pItem->taskInfo) {
@ -143,8 +146,6 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
 }

 static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
-  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
  SRSmaInfo *pRSmaInfo = NULL;

  if (!suid || !tbUids) {
@ -153,8 +154,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids)
    return TSDB_CODE_FAILED;
  }

-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, *suid);
+
+  if (!pRSmaInfo) {
    smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
    terrno = TSDB_CODE_RSMA_INVALID_STAT;
    return TSDB_CODE_FAILED;
@ -252,15 +254,14 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
  return TSDB_CODE_SUCCESS;
 }

-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *pReadHandle,
-                                       int8_t idx) {
+static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                       SReadHandle *pReadHandle, int8_t idx) {
  SRetention *pRetention = SMA_RETENTION(pSma);
  STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);

  if (param->qmsg[idx]) {
    SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
-    pItem->refId = RSMA_REF_ID(pRSmaInfo->pStat);
-    pItem->pRsmaInfo = pRSmaInfo;
+    pItem->refId = RSMA_REF_ID(pStat);
    pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
    if (!pItem->taskInfo) {
      terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
@ -348,14 +349,13 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
    goto _err;
  }
  pRSmaInfo->pTSchema = pTSchema;
-  pRSmaInfo->pStat = pStat;
  pRSmaInfo->suid = suid;

-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 0) < 0) {
    goto _err;
  }

-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 1) < 0) {
    goto _err;
  }

@ -367,7 +367,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con

  return TSDB_CODE_SUCCESS;
 _err:
-  tdFreeRSmaInfo(pRSmaInfo);
+  tdFreeRSmaInfo(pSma, pRSmaInfo);
  taosMemoryFree(pReader);
  return TSDB_CODE_FAILED;
 }
@ -538,10 +538,10 @@ int64_t tdRSmaGetMaxSubmitVer(SSma *pSma, int8_t level) {
  return atomic_load_64(&pRSmaStat->submitVer);
 }

-static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
-  SArray    *pResult = NULL;
-  SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
-  SSma      *pSma = RSMA_INFO_SMA(pRSmaInfo);
+static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                          int8_t blkType) {
+  SArray *pResult = NULL;
+  SSma   *pSma = pStat->pSma;

  while (1) {
    SSDataBlock *output = NULL;
@ -573,16 +573,16 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
    STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
    SSubmitReq *pReq = NULL;
    // TODO: the schema update should be handled
-    if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) {
+    if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) {
      smaError("vgId:%d, build submit req for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
      goto _err;
    }

-    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) {
+    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pStat->submitVer, 1), pReq) < 0) {
      taosMemoryFreeClear(pReq);
      smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
      goto _err;
    }

@ -600,84 +600,16 @@ _err:
  return TSDB_CODE_FAILED;
 }

-/**
- * @brief trigger to get rsma result
- *
- * @param param
- * @param tmrId
- */
-static void tdRSmaFetchTrigger(void *param, void *tmrId) {
-  SRSmaInfoItem *pItem = param;
-  SSma          *pSma = NULL;
-  SRSmaStat     *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
-
-  if (!pStat) {
-    smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId,
-             pItem->refId);
-    return;
-  }
-
-  pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
-
-  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
-  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
-  switch (rsmaTriggerStat) {
-    case TASK_TRIGGER_STAT_PAUSED:
-    case TASK_TRIGGER_STAT_CANCELLED:
-    case TASK_TRIGGER_STAT_FINISHED: {
-      tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is %" PRIi8
-               ", rsetId rsetId:%" PRIi64 " refId:%d",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId);
-      return;
-    }
-    default:
-      break;
-  }
-
-  int8_t fetchTriggerStat =
-      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
-  switch (fetchTriggerStat) {
-    case TASK_TRIGGER_STAT_ACTIVE: {
-      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-
-      tdRefSmaStat(pSma, (SSmaStat *)pStat);
-
-      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
-      tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
-
-      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
-    } break;
-    case TASK_TRIGGER_STAT_PAUSED: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INACTIVE: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INIT: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    default: {
-      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
-              SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-  }
-
-_end:
-  tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
-}
-
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
-                                 int8_t level) {
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
+                                 STSchema *pTSchema, tb_uid_t suid, int8_t level) {
  if (!pItem || !pItem->taskInfo) {
    smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
    return TSDB_CODE_SUCCESS;
  }
+  if (!pTSchema) {
+    smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+    return TSDB_CODE_FAILED;
+  }

  smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
           pItem->taskInfo, suid);
@ -687,14 +619,14 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
    return TSDB_CODE_FAILED;
  }

-  tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT);
-  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
-
  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
  SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);

-  if (pStat->tmrHandle) {
-    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId);
+  tdRSmaFetchAndSubmitResult(pItem, pTSchema, suid, pStat, STREAM_INPUT__DATA_SUBMIT);
+  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+
+  if (smaMgmt.tmrHandle) {
+    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
  } else {
    ASSERT(0);
  }
@ -702,19 +634,29 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
  return TSDB_CODE_SUCCESS;
 }

-static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
-  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = NULL;
  if (!pEnv) {
    // only applicable when rsma env exists
-    return TSDB_CODE_SUCCESS;
+    return NULL;
  }

-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
-  SRSmaInfo *pRSmaInfo = NULL;
-
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  if (!pStat || !RSMA_INFO_HASH(pStat)) {
+    return NULL;
+  }

+  SRSmaInfo *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+    return NULL;
+  }
+  return pRSmaInfo;
+}
+
+static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
+  if (!pRSmaInfo) {
    smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
    return TSDB_CODE_SUCCESS;
  }
@ -725,8 +667,8 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
  }

  if (inputType == STREAM_INPUT__DATA_SUBMIT) {
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1);
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
  }

  return TSDB_CODE_SUCCESS;
@ -939,13 +881,11 @@ _err:
 }

 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) {
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
  SRSmaInfo *pRSmaInfo = NULL;
  void      *qTaskInfo = NULL;

-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pItem->suid, sizeof(pItem->suid));
-
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pItem->suid);
+  if (!pRSmaInfo) {
    smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid);
    return TSDB_CODE_SUCCESS;
  }
@ -1350,3 +1290,79 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
  }
  taosReleaseRef(smaMgmt.rsetId, rsmaStat->refId);
 }
+
+/**
+ * @brief trigger to get rsma result
+ *
+ * @param param
+ * @param tmrId
+ */
+static void tdRSmaFetchTrigger(void *param, void *tmrId) {
+  SRSmaInfoItem *pItem = param;
+  SSma          *pSma = NULL;
+  SRSmaStat     *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+
+  if (!pStat) {
+    smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId,
+             pItem->refId);
+    return;
+  }
+
+  pSma = pStat->pSma;
+
+  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
+  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED:
+    case TASK_TRIGGER_STAT_CANCELLED:
+    case TASK_TRIGGER_STAT_FINISHED: {
+      tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64
+               " refId:%d",
+               SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId);
+      return;
+    }
+    default:
+      break;
+  }
+
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
+
+  ASSERT(pRSmaInfo->suid > 0);
+
+  int8_t fetchTriggerStat =
+      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
+  switch (fetchTriggerStat) {
+    case TASK_TRIGGER_STAT_ACTIVE: {
+      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+
+      tdRefSmaStat(pSma, (SSmaStat *)pStat);
+
+      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
+      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
+      tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
+
+      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+    } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INACTIVE: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INIT: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+    } break;
+    default: {
+      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
+              SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+  }
+
+_end:
+  tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+}
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@ -112,7 +112,8 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq,
  };
  tmsgSendRsp(&resp);

-  tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64 ", rspOffset:%" PRId64,
+  tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64
+          ", rspOffset:%" PRId64,
          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset);

  return 0;
@ -179,8 +180,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
  tDecoderClear(&decoder);

  if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, offset.subKey,
-            TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
+    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
+            offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
  } else if (offset.val.type == TMQ_OFFSET__LOG) {
    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey,
            TD_VID(pTq->pVnode), offset.val.version);
@ -271,6 +272,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
  tqDebug("tmq poll: consumer %ld (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId,
          pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);

+  SMqDataRsp dataRsp = {0};
+  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+
  // 2.reset offset if needed
  if (reqOffset.type > 0) {
    fetchOffsetNew = reqOffset;
@ -294,50 +298,36 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
          tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
        }
      } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
-        tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal));
+        tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
        tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %ld", consumerId, pHandle->subKey,
-                fetchOffsetNew.version);
-        SMqDataRsp dataRsp = {0};
-        tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
-        dataRsp.rspOffset = fetchOffsetNew;
-        code = 0;
+                dataRsp.rspOffset.version);
        if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
          code = -1;
        }
-        taosArrayDestroy(dataRsp.blockDataLen);
-        taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree);
-
-        if (dataRsp.withSchema) {
-          taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
-        }
-
-        if (dataRsp.withTbName) {
-          taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree);
-        }
-        return code;
+        goto OVER;
      } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
        tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64
                " in vg %d, subkey %s, reset none failed",
                pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
        terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-        return -1;
+        code = -1;
+        goto OVER;
      }
    }
  }

  // 3.query
-  SMqDataRsp dataRsp = {0};
-  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
-
-  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) {
-    fetchOffsetNew.version++;
-    if (tqScanLog(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) {
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+    if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+      fetchOffsetNew.version++;
+    }
+    if (tqScan(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) {
      ASSERT(0);
      code = -1;
      goto OVER;
    }
    if (dataRsp.blockNum == 0) {
-      // TODO add to async task
+      // TODO add to async task pool
      /*dataRsp.rspOffset.version--;*/
    }
    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
@ -346,11 +336,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
    goto OVER;
  }

-  if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+  if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) {
    int64_t     fetchVer = fetchOffsetNew.version + 1;
    SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
    if (pCkHead == NULL) {
-      return -1;
+      code = -1;
+      goto OVER;
    }

    walSetReaderCapacity(pHandle->pWalReader, 2048);
@ -403,7 +394,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
      } else {
        ASSERT(pHandle->fetchMeta);
        ASSERT(IS_META_MSG(pHead->msgType));
-        tqInfo("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
+        tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
        SMqMetaRsp metaRsp = {0};
        /*metaRsp.reqOffset = pReq->reqOffset.version;*/
        metaRsp.rspOffset = fetchVer;
@ -423,6 +414,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
    }

    taosMemoryFree(pCkHead);
+#if 0
  } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
    tqInfo("retrieve using snapshot actual offset: uid %" PRId64 " ts %" PRId64, fetchOffsetNew.uid, fetchOffsetNew.ts);
    if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
@ -433,6 +425,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
      code = -1;
    }
+#endif
  } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) {
    ASSERT(0);
  }
@ -490,6 +483,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
    /*for (int32_t i = 0; i < 5; i++) {*/
    /*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/
    /*}*/
+    int64_t ver = walGetCommittedVer(pTq->pVnode->pWal);
    if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
      pHandle->execHandle.execCol.qmsg = req.qmsg;
      req.qmsg = NULL;
@ -500,6 +494,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
            .vnode = pTq->pVnode,
            .initTableReader = true,
            .initTqReader = true,
+            .version = ver,
        };
        pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
        ASSERT(pHandle->execHandle.execCol.task[i]);
@ -508,6 +503,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
        ASSERT(scanner);
        pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner);
        ASSERT(pHandle->execHandle.pExecReader[i]);
+        pHandle->execHandle.tsdbEndVer = ver;
      }
    } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
      for (int32_t i = 0; i < 5; i++) {
@ -516,9 +512,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
      pHandle->execHandle.execDb.pFilterOutTbUid =
          taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
    } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
-      for (int32_t i = 0; i < 5; i++) {
-        pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
-      }
      pHandle->execHandle.execTb.suid = req.suid;
      SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
      vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList);
@ -528,6 +521,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
        tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid);
      }
      for (int32_t i = 0; i < 5; i++) {
+        pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
        tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList);
      }
      taosArrayDestroy(tbUidList);
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@ -59,15 +59,17 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
  return 0;
 }

-int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
  qTaskInfo_t task = pExec->execCol.task[0];

-  if (qStreamPrepareScan1(task, pOffset) < 0) {
+  if (qStreamPrepareScan(task, pOffset) < 0) {
+    ASSERT(pOffset->type == TMQ_OFFSET__LOG);
    pRsp->rspOffset = *pOffset;
    pRsp->rspOffset.version--;
    return 0;
  }

+  int32_t rowCnt = 0;
  while (1) {
    SSDataBlock* pDataBlock = NULL;
    uint64_t     ts = 0;
@ -77,11 +79,26 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff

    if (pDataBlock != NULL) {
      tqAddBlockDataToRsp(pDataBlock, pRsp);
-      if (pRsp->withTbName) {
-        int64_t uid = pExec->pExecReader[0]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp);
-      }
      pRsp->blockNum++;
+      if (pRsp->withTbName) {
+        if (pOffset->type == TMQ_OFFSET__LOG) {
+          int64_t uid = pExec->pExecReader[0]->msgIter.uid;
+          tqAddTbNameToRsp(pTq, uid, pRsp);
+        } else {
+          pRsp->withTbName = 0;
+        }
+      }
+      if (pOffset->type == TMQ_OFFSET__LOG) {
+        continue;
+      } else {
+        rowCnt += pDataBlock->info.rows;
+        if (rowCnt <= 4096) continue;
+      }
+    }
+
+    if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+      tqOffsetResetToLog(pOffset, pExec->tsdbEndVer + 1);
+      qStreamPrepareScan(task, pOffset);
      continue;
    }

@ -94,26 +111,23 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff
      ASSERT(0);
    }

-    if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) {
+    ASSERT(pRsp->rspOffset.type != 0);
+
+    if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
      ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version);
    }

-    ASSERT(pRsp->rspOffset.type != 0);
-
    break;
  }

  return 0;
 }

+#if 0
 int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) {
  ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
  qTaskInfo_t task = pExec->execCol.task[workerId];

-  /*if (qStreamScanSnapshot(task) < 0) {*/
-  /*ASSERT(0);*/
-  /*}*/
-
  if (qStreamPrepareTsdbScan(task, offset.uid, offset.ts) < 0) {
    ASSERT(0);
  }
@ -157,30 +171,12 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S

  return 0;
 }
+#endif

 int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) {
-  if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) {
-    qTaskInfo_t task = pExec->execCol.task[workerId];
-    ASSERT(task);
-    qSetStreamInput(task, pReq, STREAM_INPUT__DATA_SUBMIT, false);
-    while (1) {
-      SSDataBlock* pDataBlock = NULL;
-      uint64_t     ts = 0;
-      if (qExecTask(task, &pDataBlock, &ts) < 0) {
-        ASSERT(0);
-      }
-      if (pDataBlock == NULL) break;
+  ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN);

-      ASSERT(pDataBlock->info.rows != 0);
-
-      tqAddBlockDataToRsp(pDataBlock, pRsp);
-      if (pRsp->withTbName) {
-        int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp);
-      }
-      pRsp->blockNum++;
-    }
-  } else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
+  if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
    pRsp->withSchema = 1;
    STqReader* pReader = pExec->pExecReader[workerId];
    tqReaderSetDataMsg(pReader, pReq, 0);
@ -217,9 +213,11 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
      pRsp->blockNum++;
    }
  }
+
  if (pRsp->blockNum == 0) {
    pRsp->skipLogNum++;
    return -1;
  }
+
  return 0;
 }
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@ -22,8 +22,8 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea

  while (1) {
    if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
-      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return", pHandle->consumerId,
-              pHandle->epoch, TD_VID(pTq->pVnode), offset);
+      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
+              pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset);
      *fetchOffset = offset - 1;
      code = -1;
      goto END;
@ -104,8 +104,13 @@ void tqCloseReader(STqReader* pReader) {
 }

 int32_t tqSeekVer(STqReader* pReader, int64_t ver) {
-  //
-  return walReadSeekVer(pReader->pWalReader, ver);
+  if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
+    ASSERT(pReader->pWalReader->curInvalid);
+    ASSERT(pReader->pWalReader->curVersion == ver);
+    return -1;
+  }
+  ASSERT(pReader->pWalReader->curVersion == ver);
+  return 0;
 }

 int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
@ -114,9 +119,11 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
  while (1) {
    if (!fromProcessedMsg) {
      if (walNextValidMsg(pReader->pWalReader) < 0) {
+        pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curInvalid;
        ret->offset.type = TMQ_OFFSET__LOG;
        ret->offset.version = pReader->ver;
        ret->fetchType = FETCH_TYPE__NONE;
+        ASSERT(ret->offset.version >= 0);
        return -1;
      }
      void* body = pReader->pWalReader->pHead->head.body;
@ -131,19 +138,12 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
    }

    while (tqNextDataBlock(pReader)) {
+      // TODO mem free
      memset(&ret->data, 0, sizeof(SSDataBlock));
      int32_t code = tqRetrieveDataBlock(&ret->data, pReader);
      if (code != 0 || ret->data.info.rows == 0) {
        ASSERT(0);
        continue;
-#if 0
-        if (fromProcessedMsg) {
-          ret->fetchType = FETCH_TYPE__NONE;
-          return 0;
-        } else {
-          break;
-        }
-#endif
      }
      ret->fetchType = FETCH_TYPE__DATA;
      return 0;
@ -152,7 +152,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
    if (fromProcessedMsg) {
      ret->offset.type = TMQ_OFFSET__LOG;
      ret->offset.version = pReader->ver;
-      ASSERT(pReader->ver != -1);
+      ASSERT(pReader->ver >= 0);
      ret->fetchType = FETCH_TYPE__NONE;
      return 0;
    }
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@ -173,20 +173,64 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST
  return code;
 }

-int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row) {
+typedef struct {
+  TSKEY   ts;
+  SColVal colVal;
+} SLastCol;
+
+int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb) {
  int32_t code = 0;
  STSRow *cacheRow = NULL;
  char    key[32] = {0};
  int     keyLen = 0;

-  ((void)(row));
+  // ((void)(row));

  // getTableCacheKey(uid, "l", key, &keyLen);
  getTableCacheKey(uid, 1, key, &keyLen);
  LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen);
  if (h) {
+    STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
+    TSKEY     keyTs = row->ts;
+    bool      invalidate = false;
+
+    SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h);
+    int16_t nCol = taosArrayGetSize(pLast);
+    int16_t iCol = 0;
+
+    SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
+    if (keyTs > tTsVal->ts) {
+      STColumn *pTColumn = &pTSchema->columns[0];
+      SColVal   tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs});
+
+      taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal});
+    }
+
+    for (++iCol; iCol < nCol; ++iCol) {
+      SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
+      if (keyTs >= tTsVal->ts) {
+        SColVal *tColVal = &tTsVal->colVal;
+
+        SColVal colVal = {0};
+        tTSRowGetVal(row, pTSchema, iCol, &colVal);
+        if (colVal.isNone || colVal.isNull) {
+          if (keyTs == tTsVal->ts && !tColVal->isNone && !tColVal->isNull) {
+            invalidate = true;
+
+            break;
+          }
+        } else {
+          taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = colVal});
+        }
+      }
+    }
+
+    taosMemoryFreeClear(pTSchema);
+
+    taosLRUCacheRelease(pCache, h, invalidate);
+
    // clear last cache anyway, lazy load when get last lookup
-    taosLRUCacheRelease(pCache, h, true);
+    // taosLRUCacheRelease(pCache, h, true);
  }

  return code;
@ -516,12 +560,46 @@ typedef struct SMemNextRowIter {
  SMEMNEXTROWSTATES state;
  STbData          *pMem;  // [input]
  STbDataIter       iter;  // mem buffer skip list iterator
+  // bool              iterOpened;
+  // TSDBROW          *curRow;
 } SMemNextRowIter;

 static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow) {
+  // static int32_t getNextRowFromMem(void *iter, SArray *pRowArray) {
  SMemNextRowIter *state = (SMemNextRowIter *)iter;
  int32_t          code = 0;
+  /*
+  if (!state->iterOpened) {
+    if (state->pMem != NULL) {
+      tsdbTbDataIterOpen(state->pMem, NULL, 1, &state->iter);

+      state->iterOpened = true;
+
+      TSDBROW *pMemRow = tsdbTbDataIterGet(&state->iter);
+      if (pMemRow) {
+        state->curRow = pMemRow;
+      } else {
+        return code;
+      }
+    } else {
+      return code;
+    }
+  }
+
+  taosArrayPush(pRowArray, state->curRow);
+  while (tsdbTbDataIterNext(&state->iter)) {
+    TSDBROW *row = tsdbTbDataIterGet(&state->iter);
+
+    if (TSDBROW_TS(row) < TSDBROW_TS(state->curRow)) {
+      state->curRow = row;
+      break;
+    } else {
+      taosArrayPush(pRowArray, row);
+    }
+  }
+
+  return code;
+  */
  switch (state->state) {
    case SMEMNEXTROW_ENTER: {
      if (state->pMem != NULL) {
@ -599,7 +677,7 @@ _exit:
  return code;
 }

-static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int *iSkyline) {
+static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
  bool deleted = false;
  while (*iSkyline > 0) {
    TSDBKEY *pItemBack = (TSDBKEY *)taosArrayGet(pSkyline, *iSkyline);
@ -626,9 +704,11 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int *iSkyline) {
 }

 typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow);
+// typedef int32_t (*_next_row_fn_t)(void *iter, SArray *pRowArray);
 typedef int32_t (*_next_row_clear_fn_t)(void *iter);

-typedef struct TsdbNextRowState {
+// typedef struct TsdbNextRowState {
+typedef struct {
  TSDBROW             *pRow;
  bool                 stop;
  bool                 next;
@ -637,6 +717,388 @@ typedef struct TsdbNextRowState {
  _next_row_clear_fn_t nextRowClearFn;
 } TsdbNextRowState;

+typedef struct {
+  // STsdb  *pTsdb;
+  SArray *pSkyline;
+  int64_t iSkyline;
+
+  SBlockIdx       idx;
+  SMemNextRowIter memState;
+  SMemNextRowIter imemState;
+  SFSNextRowIter  fsState;
+  TSDBROW         memRow, imemRow, fsRow;
+
+  TsdbNextRowState input[3];
+} CacheNextRowIter;
+
+static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) {
+  int code = 0;
+
+  tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
+
+  STbData *pMem = NULL;
+  if (pTsdb->mem) {
+    tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem);
+  }
+
+  STbData *pIMem = NULL;
+  if (pTsdb->imem) {
+    tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem);
+  }
+
+  pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
+
+  SDelIdx delIdx;
+
+  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
+  if (pDelFile) {
+    SDelFReader *pDelFReader;
+
+    code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
+    if (code) goto _err;
+
+    code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
+    if (code) goto _err;
+
+    code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline);
+    if (code) goto _err;
+
+    tsdbDelFReaderClose(&pDelFReader);
+  } else {
+    code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline);
+    if (code) goto _err;
+  }
+
+  pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1;
+
+  pIter->idx = (SBlockIdx){.suid = suid, .uid = uid};
+
+  pIter->fsState.state = SFSNEXTROW_FS;
+  pIter->fsState.pTsdb = pTsdb;
+  pIter->fsState.pBlockIdxExp = &pIter->idx;
+
+  pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL};
+  pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, &pIter->imemState, getNextRowFromMem, NULL};
+  pIter->input[2] =
+      (TsdbNextRowState){&pIter->fsRow, false, true, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS};
+
+  if (pMem) {
+    pIter->memState.pMem = pMem;
+    pIter->memState.state = SMEMNEXTROW_ENTER;
+    pIter->input[0].stop = false;
+    pIter->input[0].next = true;
+  }
+
+  if (pIMem) {
+    pIter->imemState.pMem = pIMem;
+    pIter->imemState.state = SMEMNEXTROW_ENTER;
+    pIter->input[1].stop = false;
+    pIter->input[1].next = true;
+  }
+
+  return code;
+_err:
+  return code;
+}
+
+static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
+  int code = 0;
+
+  for (int i = 0; i < 3; ++i) {
+    if (pIter->input[i].nextRowClearFn) {
+      pIter->input[i].nextRowClearFn(pIter->input[i].iter);
+    }
+  }
+
+  if (pIter->pSkyline) {
+    taosArrayDestroy(pIter->pSkyline);
+  }
+
+  return code;
+_err:
+  return code;
+}
+
+// iterate next row non deleted backward ts, version (from high to low)
+static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) {
+  int code = 0;
+
+  for (int i = 0; i < 3; ++i) {
+    if (pIter->input[i].next && !pIter->input[i].stop) {
+      code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow);
+      if (code) goto _err;
+
+      if (pIter->input[i].pRow == NULL) {
+        pIter->input[i].stop = true;
+        pIter->input[i].next = false;
+      }
+    }
+  }
+
+  if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop) {
+    *ppRow = NULL;
+    return code;
+  }
+
+  // select maxpoint(s) from mem, imem, fs
+  TSDBROW *max[3] = {0};
+  int      iMax[3] = {-1, -1, -1};
+  int      nMax = 0;
+  TSKEY    maxKey = TSKEY_MIN;
+
+  for (int i = 0; i < 3; ++i) {
+    if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) {
+      TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow);
+
+      // merging & deduplicating on client side
+      if (maxKey <= key.ts) {
+        if (maxKey < key.ts) {
+          nMax = 0;
+          maxKey = key.ts;
+        }
+
+        iMax[nMax] = i;
+        max[nMax++] = pIter->input[i].pRow;
+      }
+    }
+  }
+
+  // delete detection
+  TSDBROW *merge[3] = {0};
+  int      iMerge[3] = {-1, -1, -1};
+  int      nMerge = 0;
+  for (int i = 0; i < nMax; ++i) {
+    TSDBKEY maxKey = TSDBROW_KEY(max[i]);
+
+    bool deleted = tsdbKeyDeleted(&maxKey, pIter->pSkyline, &pIter->iSkyline);
+    if (!deleted) {
+      iMerge[nMerge] = iMax[i];
+      merge[nMerge++] = max[i];
+    }
+
+    pIter->input[iMax[i]].next = deleted;
+  }
+
+  if (nMerge > 0) {
+    pIter->input[iMerge[0]].next = true;
+
+    *ppRow = merge[0];
+  } else {
+    *ppRow = NULL;
+  }
+
+  return code;
+_err:
+  return code;
+}
+
+static int32_t mergeLastRow2(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRow) {
+  int32_t code = 0;
+
+  STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
+  int16_t   nCol = pTSchema->numOfCols;
+  int16_t   iCol = 0;
+  int16_t   noneCol = 0;
+  bool      setNoneCol = false;
+  SArray   *pColArray = taosArrayInit(nCol, sizeof(SColVal));
+  SColVal  *pColVal = &(SColVal){0};
+
+  // tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
+
+  TSKEY lastRowTs = TSKEY_MAX;
+
+  CacheNextRowIter iter = {0};
+  nextRowIterOpen(&iter, uid, pTsdb);
+
+  do {
+    TSDBROW *pRow = NULL;
+    nextRowIterGet(&iter, &pRow);
+
+    if (!pRow) {
+      break;
+    }
+
+    if (lastRowTs == TSKEY_MAX) {
+      lastRowTs = TSDBROW_TS(pRow);
+      STColumn *pTColumn = &pTSchema->columns[0];
+
+      *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs});
+      if (taosArrayPush(pColArray, pColVal) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+
+      for (iCol = 1; iCol < nCol; ++iCol) {
+        tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+
+        if (taosArrayPush(pColArray, pColVal) == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
+
+        if (pColVal->isNone && !setNoneCol) {
+          noneCol = iCol;
+          setNoneCol = true;
+        }
+      }
+      if (!setNoneCol) {
+        // goto build the result ts row
+        break;
+      } else {
+        continue;
+      }
+    }
+
+    if ((TSDBROW_TS(pRow) < lastRowTs)) {
+      // goto build the result ts row
+      break;
+    }
+
+    // merge into pColArray
+    setNoneCol = false;
+    for (iCol = noneCol; iCol < nCol; ++iCol) {
+      // high version's column value
+      SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol);
+
+      tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+      if (tColVal->isNone && !pColVal->isNone) {
+        taosArraySet(pColArray, iCol, pColVal);
+      } else if (tColVal->isNone && pColVal->isNone && !setNoneCol) {
+        noneCol = iCol;
+        setNoneCol = true;
+      }
+    }
+  } while (setNoneCol);
+
+  // build the result ts row here
+  *dup = false;
+  if (taosArrayGetSize(pColArray) == nCol) {
+    code = tdSTSRowNew(pColArray, pTSchema, ppRow);
+    if (code) goto _err;
+  } else {
+    *ppRow = NULL;
+  }
+
+  nextRowIterClose(&iter);
+  taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+
+_err:
+  nextRowIterClose(&iter);
+  taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+}
+
+static int32_t mergeLast2(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
+  int32_t code = 0;
+
+  STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
+  int16_t   nCol = pTSchema->numOfCols;
+  int16_t   iCol = 0;
+  int16_t   noneCol = 0;
+  bool      setNoneCol = false;
+  SArray   *pColArray = taosArrayInit(nCol, sizeof(SLastCol));
+  SColVal  *pColVal = &(SColVal){0};
+
+  // tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
+
+  TSKEY lastRowTs = TSKEY_MAX;
+
+  CacheNextRowIter iter = {0};
+  nextRowIterOpen(&iter, uid, pTsdb);
+
+  do {
+    TSDBROW *pRow = NULL;
+    nextRowIterGet(&iter, &pRow);
+
+    if (!pRow) {
+      break;
+    }
+
+    TSKEY rowTs = TSDBROW_TS(pRow);
+
+    if (lastRowTs == TSKEY_MAX) {
+      lastRowTs = rowTs;
+      STColumn *pTColumn = &pTSchema->columns[0];
+
+      *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs});
+      if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+
+      for (iCol = 1; iCol < nCol; ++iCol) {
+        tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+
+        if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
+
+        if ((pColVal->isNone || pColVal->isNull) && !setNoneCol) {
+          noneCol = iCol;
+          setNoneCol = true;
+        }
+      }
+      if (!setNoneCol) {
+        // goto build the result ts row
+        break;
+      } else {
+        continue;
+      }
+    }
+    /*
+    if ((TSDBROW_TS(pRow) < lastRowTs)) {
+      // goto build the result ts row
+      break;
+    }
+    */
+    // merge into pColArray
+    setNoneCol = false;
+    for (iCol = noneCol; iCol < nCol; ++iCol) {
+      // high version's column value
+      SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol);
+
+      tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+      if ((tColVal->isNone || tColVal->isNull) && (!pColVal->isNone && !pColVal->isNull)) {
+        taosArraySet(pColArray, iCol, &(SLastCol){.ts = rowTs, .colVal = *pColVal});
+        //} else if (tColVal->isNone && pColVal->isNone && !setNoneCol) {
+      } else if ((tColVal->isNone || tColVal->isNull) && (pColVal->isNone || pColVal->isNull) && !setNoneCol) {
+        noneCol = iCol;
+        setNoneCol = true;
+      }
+    }
+  } while (setNoneCol);
+
+  // build the result ts row here
+  //*dup = false;
+  if (taosArrayGetSize(pColArray) <= 0) {
+    *ppLastArray = NULL;
+    taosArrayDestroy(pColArray);
+  } else {
+    *ppLastArray = pColArray;
+  }
+  /*  if (taosArrayGetSize(pColArray) == nCol) {
+    code = tdSTSRowNew(pColArray, pTSchema, ppRow);
+    if (code) goto _err;
+  } else {
+    *ppRow = NULL;
+    }*/
+
+  nextRowIterClose(&iter);
+  // taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+
+_err:
+  nextRowIterClose(&iter);
+  // taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+}
+
 static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRow) {
  int32_t code = 0;
  SArray *pSkyline = NULL;
@ -682,7 +1144,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
    if (code) goto _err;
  }

-  int iSkyline = taosArrayGetSize(pSkyline) - 1;
+  int64_t iSkyline = taosArrayGetSize(pSkyline) - 1;

  SBlockIdx idx = {.suid = suid, .uid = uid};

@ -719,12 +1181,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
  do {
    for (int i = 0; i < 3; ++i) {
      if (input[i].next && !input[i].stop) {
-        code = input[i].nextRowFn(input[i].iter, &input[i].pRow);
-        if (code) goto _err;
-
        if (input[i].pRow == NULL) {
-          input[i].stop = true;
-          input[i].next = false;
+          code = input[i].nextRowFn(input[i].iter, &input[i].pRow);
+          if (code) goto _err;
+
+          if (input[i].pRow == NULL) {
+            input[i].stop = true;
+            input[i].next = false;
+          }
        }
      }
    }
@ -758,14 +1222,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo

    // delete detection
    TSDBROW *merge[3] = {0};
-    // int      iMerge[3] = {-1, -1, -1};
-    int nMerge = 0;
+    int      iMerge[3] = {-1, -1, -1};
+    int      nMerge = 0;
    for (int i = 0; i < nMax; ++i) {
      TSDBKEY maxKey = TSDBROW_KEY(max[i]);

      bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline);
      if (!deleted) {
-        // iMerge[nMerge] = i;
+        iMerge[nMerge] = i;
        merge[nMerge++] = max[i];
      }

@ -792,7 +1256,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
      }
    }

-  } while (*ppRow == NULL);
+  } while (1);

  for (int i = 0; i < 3; ++i) {
    if (input[i].nextRowClearFn) {
@ -819,11 +1283,6 @@ _err:
  return code;
 }

-typedef struct {
-  TSKEY   ts;
-  SColVal colVal;
-} SLastCol;
-
 // static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
 static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
  int32_t  code = 0;
@ -873,7 +1332,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
    if (code) goto _err;
  }

-  int iSkyline = taosArrayGetSize(pSkyline) - 1;
+  int64_t iSkyline = taosArrayGetSize(pSkyline) - 1;

  SBlockIdx idx = {.suid = suid, .uid = uid};

@ -1128,7 +1587,7 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
  } else {
    STSRow *pRow = NULL;
    bool    dup = false;  // which is always false for now
-    code = mergeLastRow(uid, pTsdb, &dup, &pRow);
+    code = mergeLastRow2(uid, pTsdb, &dup, &pRow);
    // if table's empty or error, return code of -1
    if (code < 0 || pRow == NULL) {
      if (!dup && pRow) {
@ -1195,7 +1654,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand
    // STSRow *pRow = NULL;
    // code = mergeLast(uid, pTsdb, &pRow);
    SArray *pLastArray = NULL;
-    code = mergeLast(uid, pTsdb, &pLastArray);
+    // code = mergeLast(uid, pTsdb, &pLastArray);
+    code = mergeLast2(uid, pTsdb, &pLastArray);
    // if table's empty or error, return code of -1
    // if (code < 0 || pRow == NULL) {
    if (code < 0 || pLastArray == NULL) {
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@ -561,7 +561,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
    }
  }

-  tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow);
+  tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb);

  pTbData->minVersion = TMIN(pTbData->minVersion, version);
  pTbData->maxVersion = TMAX(pTbData->maxVersion, version);
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@ -59,7 +59,6 @@ typedef struct SBlockLoadSuppInfo {
  SColumnDataAgg   tsColAgg;
  SColumnDataAgg** plist;
  int16_t*         colIds;    // column ids for loading file block data
-  int32_t*         slotIds;   // colId to slotId
  char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
 } SBlockLoadSuppInfo;

@ -183,7 +182,6 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
    return NULL;
  }

-  // todo apply the lastkey of table check to avoid to load header file
  for (int32_t j = 0; j < numOfTables; ++j) {
    STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
    if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
@ -218,6 +216,30 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) {
  }
 }

+static void destroyBlockScanInfo(SHashObj* pTableMap) {
+  STableBlockScanInfo* p = NULL;
+
+  while ((p = taosHashIterate(pTableMap, p)) != NULL) {
+    p->iterInit = false;
+    p->iiter.hasVal = false;
+
+    if (p->iter.iter != NULL) {
+      tsdbTbDataIterDestroy(p->iter.iter);
+      p->iter.iter = NULL;
+    }
+
+    if (p->iiter.iter != NULL) {
+      tsdbTbDataIterDestroy(p->iiter.iter);
+      p->iiter.iter = NULL;
+    }
+
+    taosArrayDestroy(p->delSkyline);
+    p->delSkyline = NULL;
+  }
+
+  taosHashCleanup(pTableMap);
+}
+
 static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) {
  ASSERT(pWindow != NULL);
  return pWindow->skey > pWindow->ekey;
@ -265,6 +287,10 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, const STsdbFSState* pFSt
  return TSDB_CODE_SUCCESS;
 }

+static void cleanupFilesetIterator(SFilesetIter* pIter) {
+  taosArrayDestroy(pIter->pFileList);
+}
+
 static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
  bool    asc = ASCENDING_TRAVERSE(pIter->order);
  int32_t step = asc ? 1 : -1;
@ -297,6 +323,9 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {

    if ((asc && (win.ekey < pReader->window.skey)) || ((!asc) && (win.skey > pReader->window.ekey))) {
      pIter->index += step;
+      if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) {
+        return false;
+      }
      continue;
    }

@ -313,7 +342,15 @@ static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) {
  pIter->order = order;
  pIter->index = -1;
  pIter->numOfBlocks = -1;
-  pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));
+  if (pIter->blockList == NULL) {
+    pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));
+  } else {
+    taosArrayClear(pIter->blockList);
+  }
+}
+
+static void cleanupDataBlockIterator(SDataBlockIter* pIter) {
+  taosArrayDestroy(pIter->blockList);
 }

 static void initReaderStatus(SReaderStatus* pStatus) {
@ -356,14 +393,14 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
  initReaderStatus(&pReader->status);

  pReader->pTsdb =
-      getTsdbByRetentions(pVnode, pCond->twindows[0].skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
+      getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
  pReader->suid = pCond->suid;
  pReader->order = pCond->order;
  pReader->capacity = 4096;
  pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL;
  pReader->verRange = getQueryVerRange(pVnode, pCond, level);
  pReader->type = pCond->type;
-  pReader->window = updateQueryTimeWindow(pVnode->pTsdb, pCond->twindows);
+  pReader->window = updateQueryTimeWindow(pVnode->pTsdb, &pCond->twindows);

  ASSERT(pCond->numOfCols > 0);

@ -2515,6 +2552,7 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe
  tRowMergerInit(&merge, pRow, pReader->pSchema);
  doMergeRowsInBuf(pIter, k.ts, pDelList, &merge, pReader);
  tRowMergerGetRow(&merge, pTSRow);
+  tRowMergerClear(&merge);
 }

 void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader,
@ -2651,6 +2689,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
    }

    doAppendOneRow(pBlock, pReader, pTSRow);
+    taosMemoryFree(pTSRow);

    // no data in buffer, return immediately
    if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) {
@ -2778,11 +2817,24 @@ void tsdbReaderClose(STsdbReader* pReader) {
    return;
  }

-  blockDataDestroy(pReader->pResBlock);
-  taosMemoryFreeClear(pReader->suppInfo.plist);
+  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
+
+  taosMemoryFreeClear(pSupInfo->plist);
+  taosMemoryFree(pSupInfo->colIds);
+
+  taosArrayDestroy(pSupInfo->pColAgg);
+  for(int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) {
+    if (pSupInfo->buildBuf[i] != NULL) {
+      taosMemoryFreeClear(pSupInfo->buildBuf[i]);
+    }
+  }
+  taosMemoryFree(pSupInfo->buildBuf);
+
+  cleanupFilesetIterator(&pReader->status.fileIter);
+  cleanupDataBlockIterator(&pReader->status.blockIter);
+  destroyBlockScanInfo(pReader->status.pTableMap);
+  blockDataDestroy(pReader->pResBlock);

-  taosArrayDestroy(pReader->suppInfo.pColAgg);
-  taosMemoryFree(pReader->suppInfo.slotIds);

 #if 0
 //   if (pReader->status.pTableScanInfo != NULL) {
@ -2954,7 +3006,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
  return pReader->pResBlock->pDataBlock;
 }

-int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) {
+int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
  if (isEmptyQueryTimeWindow(&pReader->window)) {
    return TSDB_CODE_SUCCESS;
  }
@ -2964,7 +3016,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_
  pReader->status.loadFromFile = true;
  pReader->status.pTableIter = NULL;

-  pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows[tWinIdx]);
+  pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows);

  // allocate buffer in order to load data blocks from file
  memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));
--- a/source/dnode/vnode/src/tsdb/tsdbRetention.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@ -13,33 +13,30 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

-#include "vnd.h"
+#include "tsdb.h"

-int32_t vnodeRealloc(void** pp, int32_t size) {
-  uint8_t* p = NULL;
-  int32_t  csize = 0;
+int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
+  int32_t code = 0;

-  if (*pp) {
-    p = (uint8_t*)(*pp) - sizeof(int32_t);
-    csize = *(int32_t*)p;
+  // begin
+  code = tsdbFSBegin(pTsdb->fs);
+  if (code) goto _err;
+
+  // do retention
+  for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs->nState->aDFileSet); iSet++) {
+    SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pTsdb->fs->nState->aDFileSet, iSet);
+
+    // TODO
  }

-  if (csize >= size) {
-    return 0;
-  }
+  // commit
+  code = tsdbFSCommit(pTsdb->fs);
+  if (code) goto _err;

-  p = (uint8_t*)taosMemoryRealloc(p, size);
-  if (p == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  *(int32_t*)p = size;
-  *pp = p + sizeof(int32_t);
+_exit:
+  return code;

-  return 0;
-}
-
-void vnodeFree(void* p) {
-  if (p) {
-    taosMemoryFree(((uint8_t*)p) - sizeof(int32_t));
-  }
+_err:
+  tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
 }
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@ -15,22 +15,686 @@

 #include "tsdb.h"

-struct STsdbSnapshotReader {
-  STsdb* pTsdb;
-  // TODO
+// STsdbSnapReader ========================================
+struct STsdbSnapReader {
+  STsdb*  pTsdb;
+  int64_t sver;
+  int64_t ever;
+  // for data file
+  int8_t        dataDone;
+  int32_t       fid;
+  SDataFReader* pDataFReader;
+  SArray*       aBlockIdx;  // SArray<SBlockIdx>
+  int32_t       iBlockIdx;
+  SBlockIdx*    pBlockIdx;
+  SMapData      mBlock;  // SMapData<SBlock>
+  int32_t       iBlock;
+  SBlockData    blkData;
+  // for del file
+  int8_t       delDone;
+  SDelFReader* pDelFReader;
+  int32_t      iDelIdx;
+  SArray*      aDelIdx;   // SArray<SDelIdx>
+  SArray*      aDelData;  // SArray<SDelData>
 };

-int32_t tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  // TODO
-  return 0;
+static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t code = 0;
+
+  while (true) {
+    if (pReader->pDataFReader == NULL) {
+      SDFileSet* pSet = NULL;
+
+      // search the next data file set to read (todo)
+      if (0 /* TODO */) {
+        code = TSDB_CODE_VND_READ_END;
+        goto _exit;
+      }
+
+      // open
+      code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
+      if (code) goto _err;
+
+      // SBlockIdx
+      code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL);
+      if (code) goto _err;
+
+      pReader->iBlockIdx = 0;
+      pReader->pBlockIdx = NULL;
+    }
+
+    while (true) {
+      if (pReader->pBlockIdx == NULL) {
+        if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) {
+          tsdbDataFReaderClose(&pReader->pDataFReader);
+          break;
+        }
+
+        pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+        pReader->iBlockIdx++;
+
+        // SBlock
+        code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL);
+        if (code) goto _err;
+
+        pReader->iBlock = 0;
+      }
+
+      while (true) {
+        SBlock  block;
+        SBlock* pBlock = &block;
+
+        if (pReader->iBlock >= pReader->mBlock.nItem) {
+          pReader->pBlockIdx = NULL;
+          break;
+        }
+
+        tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock);
+        pReader->iBlock++;
+
+        if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) ||
+            (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) {
+          // overlap (todo)
+
+          code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->blkData, NULL, NULL);
+          if (code) goto _err;
+
+          goto _exit;
+        }
+      }
+    }
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d snap read data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
 }

-int32_t tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader) {
-  // TODO
-  return 0;
+static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t   code = 0;
+  STsdb*    pTsdb = pReader->pTsdb;
+  SDelFile* pDelFile = pTsdb->fs->cState->pDelFile;
+
+  if (pReader->pDelFReader == NULL) {
+    if (pDelFile == NULL) {
+      code = TSDB_CODE_VND_READ_END;
+      goto _exit;
+    }
+
+    // open
+    code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb, NULL);
+    if (code) goto _err;
+
+    // read index
+    code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx, NULL);
+    if (code) goto _err;
+
+    pReader->iDelIdx = 0;
+  }
+
+  while (pReader->iDelIdx < taosArrayGetSize(pReader->aDelIdx)) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx);
+    int32_t  size = 0;
+
+    pReader->iDelIdx++;
+
+    code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL);
+    if (code) goto _err;
+
+    for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
+      SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
+
+      if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
+        size += tPutDelData(NULL, pDelData);
+      }
+    }
+
+    if (size > 0) {
+      int64_t n = 0;
+
+      size = size + sizeof(SSnapDataHdr) + sizeof(TABLEID);
+      code = tRealloc(ppData, size);
+      if (code) goto _err;
+
+      // SSnapDataHdr
+      SSnapDataHdr* pSnapDataHdr = (SSnapDataHdr*)(*ppData + n);
+      pSnapDataHdr->type = 1;
+      pSnapDataHdr->size = size;  // TODO: size here may incorrect
+      n += sizeof(SSnapDataHdr);
+
+      // TABLEID
+      TABLEID* pId = (TABLEID*)(*ppData + n);
+      pId->suid = pDelIdx->suid;
+      pId->uid = pDelIdx->uid;
+      n += sizeof(*pId);
+
+      // DATA
+      for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
+        SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
+
+        if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
+          n += tPutDelData(*ppData + n, pDelData);
+        }
+      }
+
+      goto _exit;
+    }
+  }
+
+  code = TSDB_CODE_VND_READ_END;
+  tsdbDelFReaderClose(&pReader->pDelFReader);
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d snap read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
 }

-int32_t tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData) {
-  // TODO
-  return 0;
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader) {
+  int32_t          code = 0;
+  STsdbSnapReader* pReader = NULL;
+
+  // alloc
+  pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
+  if (pReader == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pReader->pTsdb = pTsdb;
+  pReader->sver = sver;
+  pReader->ever = ever;
+
+  pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+  if (pReader->aBlockIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pReader->mBlock = tMapDataInit();
+
+  code = tBlockDataInit(&pReader->blkData);
+  if (code) goto _err;
+
+  pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
+  if (pReader->aDelIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
+  if (pReader->aDelData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  *ppReader = pReader;
+  return code;
+
+_err:
+  tsdbError("vgId:%d snapshot reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppReader = NULL;
+  return code;
+}
+
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
+  int32_t          code = 0;
+  STsdbSnapReader* pReader = *ppReader;
+
+  taosArrayDestroy(pReader->aDelData);
+  taosArrayDestroy(pReader->aDelIdx);
+  if (pReader->pDelFReader) {
+    tsdbDelFReaderClose(&pReader->pDelFReader);
+  }
+  tBlockDataClear(&pReader->blkData);
+  tMapDataClear(&pReader->mBlock);
+  taosArrayDestroy(pReader->aBlockIdx);
+  if (pReader->pDataFReader) {
+    tsdbDataFReaderClose(&pReader->pDataFReader);
+  }
+  taosMemoryFree(pReader);
+  *ppReader = NULL;
+
+  return code;
+}
+
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t code = 0;
+
+  // read data file
+  if (!pReader->dataDone) {
+    code = tsdbSnapReadData(pReader, ppData);
+    if (code) {
+      if (code == TSDB_CODE_VND_READ_END) {
+        pReader->dataDone = 1;
+      } else {
+        goto _err;
+      }
+    } else {
+      goto _exit;
+    }
+  }
+
+  // read del file
+  if (!pReader->delDone) {
+    code = tsdbSnapReadDel(pReader, ppData);
+    if (code) {
+      if (code == TSDB_CODE_VND_READ_END) {
+        pReader->delDone = 1;
+      } else {
+        goto _err;
+      }
+    } else {
+      goto _exit;
+    }
+  }
+
+  code = TSDB_CODE_VND_READ_END;
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d snapshot read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+// STsdbSnapWriter ========================================
+struct STsdbSnapWriter {
+  STsdb*  pTsdb;
+  int64_t sver;
+  int64_t ever;
+
+  // config
+  int32_t minutes;
+  int8_t  precision;
+
+  // for data file
+  int32_t       fid;
+  SDataFReader* pDataFReader;
+  SArray*       aBlockIdx;
+  int32_t       iBlockIdx;
+  SBlockIdx*    pBlockIdx;
+  SMapData      mBlock;
+  int32_t       iBlock;
+  SBlockData    blockData;
+  int32_t       iRow;
+
+  SDataFWriter* pDataFWriter;
+  SArray*       aBlockIdxN;
+  SBlockIdx     blockIdx;
+  SMapData      mBlockN;
+  SBlock        block;
+  SBlockData    nBlockData;
+
+  // for del file
+  SDelFReader* pDelFReader;
+  SDelFWriter* pDelFWriter;
+  int32_t      iDelIdx;
+  SArray*      aDelIdx;
+  SArray*      aDelData;
+  SArray*      aDelIdxN;
+};
+
+static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDataFWriter == NULL) goto _exit;
+
+  // TODO
+
+  code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 0);
+  if (code) goto _err;
+
+  if (pWriter->pDataFReader) {
+    code = tsdbDataFReaderClose(&pWriter->pDataFReader);
+    if (code) goto _err;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t     code = 0;
+  int32_t     iRow = 0;           // todo
+  int32_t     nRow = 0;           // todo
+  SBlockData* pBlockData = NULL;  // todo
+
+  while (iRow < nRow) {
+    code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write append data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+  int64_t suid = 0;  // todo
+  int64_t uid = 0;   // todo
+  int64_t skey;      // todo
+  int64_t ekey;      // todo
+
+  int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision);
+  ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision));
+
+  // begin
+  if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
+
+    pWriter->fid = fid;
+    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid);
+    // reader
+    if (pSet) {
+      // open
+      code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
+      if (code) goto _err;
+
+      // SBlockIdx
+      code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL);
+      if (code) goto _err;
+    } else {
+      taosArrayClear(pWriter->aBlockIdx);
+    }
+    pWriter->iBlockIdx = 0;
+
+    // writer
+    SDFileSet wSet = {0};
+    if (pSet == NULL) {
+      wSet = (SDFileSet){0};  // todo
+    } else {
+      wSet = (SDFileSet){0};  // todo
+    }
+
+    code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
+    if (code) goto _err;
+
+    taosArrayClear(pWriter->aBlockIdxN);
+  }
+
+  // process
+  TABLEID id = {0};    // TODO
+  TSKEY   minKey = 0;  // TODO
+  TSKEY   maxKey = 0;  // TODO
+
+  while (true) {
+    if (pWriter->pBlockIdx) {
+      int32_t c = tTABLEIDCmprFn(&id, pWriter->pBlockIdx);
+
+      if (c == 0) {
+      } else if (c < 0) {
+        // keep merge
+      } else {
+        // code = tsdbSnapWriteTableDataEnd(pWriter);
+        if (code) goto _err;
+
+        pWriter->iBlockIdx++;
+        if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
+          pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
+        } else {
+          pWriter->pBlockIdx = NULL;
+        }
+
+        if (pWriter->pBlockIdx) {
+          code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL);
+          if (code) goto _err;
+        }
+      }
+    } else {
+      int32_t c = tTABLEIDCmprFn(&id, &pWriter->blockIdx);
+
+      if (c == 0) {
+        // merge commit the block data
+      } else if (c > 0) {
+        // code = tsdbSnapWriteTableDataEnd(pWriter);
+        if (code) goto _err;
+      } else {
+        ASSERT(0);
+      }
+    }
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDelFWriter == NULL) {
+    SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->nState);
+
+    // reader
+    if (pDelFile) {
+      code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL);
+      if (code) goto _err;
+
+      code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdx, NULL);
+      if (code) goto _err;
+    }
+
+    // writer
+    SDelFile delFile = {.commitID = pTsdb->pVnode->state.commitID, .offset = 0, .size = 0};
+    code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb);
+    if (code) goto _err;
+  }
+
+  // process the del data
+  TABLEID id = {0};  // todo
+
+  while (true) {
+    SDelIdx* pDelIdx = NULL;
+    int64_t  n = 0;
+    SDelData delData;
+    SDelIdx  delIdx;
+    int8_t   toBreak = 0;
+
+    if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx)) {
+      pDelIdx = taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+    }
+
+    if (pDelIdx) {
+      int32_t c = tTABLEIDCmprFn(&id, pDelIdx);
+      if (c < 0) {
+        goto _new_del;
+      } else {
+        code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
+        if (code) goto _err;
+
+        pWriter->iDelIdx++;
+        if (c == 0) {
+          toBreak = 1;
+          delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
+          goto _merge_del;
+        } else {
+          delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
+          goto _write_del;
+        }
+      }
+    }
+
+  _new_del:
+    toBreak = 1;
+    delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
+    taosArrayClear(pWriter->aDelData);
+
+  _merge_del:
+    while (n < nData) {
+      n += tGetDelData(pData + n, &delData);
+      if (taosArrayPush(pWriter->aDelData, &delData) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+    }
+
+  _write_del:
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
+    if (code) goto _err;
+
+    if (taosArrayPush(pWriter->aDelIdxN, &delIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+
+    if (toBreak) break;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDelFWriter == NULL) goto _exit;
+  for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx); pWriter->iDelIdx++) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+
+    code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
+    if (code) goto _err;
+
+    SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
+    if (code) goto _err;
+
+    if (taosArrayPush(pWriter->aDelIdx, &delIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+
+  code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
+  if (code) goto _err;
+
+  code = tsdbFSStateUpsertDelFile(pTsdb->fs->nState, &pWriter->pDelFWriter->fDel);
+  if (code) goto _err;
+
+  code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1);
+  if (code) goto _err;
+
+  if (pWriter->pDelFReader) {
+    code = tsdbDelFReaderClose(&pWriter->pDelFReader);
+    if (code) goto _err;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshow write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
+  int32_t          code = 0;
+  STsdbSnapWriter* pWriter = NULL;
+
+  // alloc
+  pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pTsdb = pTsdb;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  *ppWriter = pWriter;
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppWriter = NULL;
+  return code;
+}
+
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
+  int32_t          code = 0;
+  STsdbSnapWriter* pWriter = *ppWriter;
+
+  if (rollback) {
+    code = tsdbSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
+
+    code = tsdbSnapWriteDelEnd(pWriter);
+    if (code) goto _err;
+
+    code = tsdbSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+
+  taosMemoryFree(pWriter);
+  *ppWriter = NULL;
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  int8_t  type = pData[0];
+
+  // ts data
+  if (type == 0) {
+    code = tsdbSnapWriteData(pWriter, pData + 1, nData - 1);
+    if (code) goto _err;
+  } else {
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
+  }
+
+  // del data
+  if (type == 1) {
+    code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1);
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshow write failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
 }
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@ -87,8 +87,10 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {

  n += tPutI32v(p ? p + n : p, pMapData->nItem);
  if (pMapData->nItem) {
+    int32_t lOffset = 0;
    for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
-      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem]);
+      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset);
+      lOffset = pMapData->aOffset[iItem];
    }

    n += tPutI32v(p ? p + n : p, pMapData->nData);
@ -111,8 +113,11 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
  if (pMapData->nItem) {
    if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1;

+    int32_t lOffset = 0;
    for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
      n += tGetI32v(p + n, &pMapData->aOffset[iItem]);
+      pMapData->aOffset[iItem] += lOffset;
+      lOffset = pMapData->aOffset[iItem];
    }

    n += tGetI32v(p + n, &pMapData->nData);
--- a/source/dnode/vnode/src/vnd/vnodeCfg.c
+++ b/source/dnode/vnode/src/vnd/vnodeCfg.c
@ -15,30 +15,37 @@

 #include "vnd.h"

-const SVnodeCfg vnodeCfgDefault = {
-    .vgId = -1,
-    .dbname = "",
-    .dbId = 0,
-    .szPage = 4096,
-    .szCache = 256,
-    .szBuf = 96 * 1024 * 1024,
-    .isHeap = false,
-    .isWeak = 0,
-    .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
-                .update = 1,
-                .compression = 2,
-                .slLevel = 5,
-                .days = 14400,
-                .minRows = 100,
-                .maxRows = 4096,
-                .keep2 = 5256000,
-                .keep0 = 5256000,
-                .keep1 = 5256000},
-    .walCfg =
-        {.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE},
-    .hashBegin = 0,
-    .hashEnd = 0,
-    .hashMethod = 0};
+const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
+                                   .dbname = "",
+                                   .dbId = 0,
+                                   .szPage = 4096,
+                                   .szCache = 256,
+                                   .szBuf = 96 * 1024 * 1024,
+                                   .isHeap = false,
+                                   .isWeak = 0,
+                                   .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
+                                               .update = 1,
+                                               .compression = 2,
+                                               .slLevel = 5,
+                                               .days = 14400,
+                                               .minRows = 100,
+                                               .maxRows = 4096,
+                                               .keep2 = 5256000,
+                                               .keep0 = 5256000,
+                                               .keep1 = 5256000},
+                                   .walCfg =
+                                       {
+                                           .vgId = -1,
+                                           .fsyncPeriod = 0,
+                                           .retentionPeriod = -1,
+                                           .rollPeriod = -1,
+                                           .segSize = -1,
+                                           .retentionSize = -1,
+                                           .level = TAOS_WAL_WRITE,
+                                       },
+                                   .hashBegin = 0,
+                                   .hashEnd = 0,
+                                   .hashMethod = 0};

 int vnodeCheckCfg(const SVnodeCfg *pCfg) {
  // TODO
@ -79,7 +86,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
    SJson *pNodeRetentions = tjsonCreateArray();
    tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
    for (int32_t i = 0; i < nRetention; ++i) {
-      SJson *           pNodeRetention = tjsonCreateObject();
+      SJson            *pNodeRetention = tjsonCreateObject();
      const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
      tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq);
      tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit);
@ -156,7 +163,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
  if (code < 0) return -1;
  tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
  if (code < 0) return -1;
-  SJson * pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
+  SJson  *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
  int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
  if (nRetention > TSDB_RETENTION_MAX) {
    nRetention = TSDB_RETENTION_MAX;
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@ -15,7 +15,7 @@

 #include "vnd.h"

-#define VND_INFO_FNAME     "vnode.json"
+#define VND_INFO_FNAME "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"

 static int  vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
@ -223,12 +223,14 @@ int vnodeCommit(SVnode *pVnode) {
  // save info
  info.config = pVnode->config;
  info.state.committed = pVnode->state.applied;
+  info.state.commitTerm = pVnode->state.applyTerm;
  info.state.commitID = pVnode->state.commitID;
  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
  if (vnodeSaveInfo(dir, &info) < 0) {
    ASSERT(0);
    return -1;
  }
+  walBeginSnapshot(pVnode->pWal, pVnode->state.applied);

  // preCommit
  smaPreCommit(pVnode->pSma);
@ -277,6 +279,7 @@ int vnodeCommit(SVnode *pVnode) {
  smaPostCommit(pVnode->pSma);

  // apply the commit (TODO)
+  walEndSnapshot(pVnode->pWal);
  vnodeBufPoolReset(pVnode->onCommit);
  pVnode->onCommit->next = pVnode->pPool;
  pVnode->pPool = pVnode->onCommit;
@ -316,6 +319,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {

  if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
  if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1;

  return 0;
 }
@ -328,6 +332,8 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
  if (code < 0) return -1;
  tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
  if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
+  if (code < 0) return -1;

  return 0;
 }
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@ -79,8 +79,10 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
  strcpy(pVnode->path, path);
  pVnode->config = info.config;
  pVnode->state.committed = info.state.committed;
+  pVnode->state.commitTerm = info.state.commitTerm;
  pVnode->state.applied = info.state.committed;
  pVnode->state.commitID = info.state.commitID;
+  pVnode->state.commitTerm = info.state.commitTerm;
  pVnode->pTfs = pTfs;
  pVnode->msgCb = msgCb;
  pVnode->blockCount = 0;
@ -115,6 +117,13 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
  // open wal
  sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR);
  taosRealPath(tdir, NULL, sizeof(tdir));
+
+// for test tsdb snapshot
+#if 0
+  pVnode->config.walCfg.segSize = 200;
+  pVnode->config.walCfg.retentionSize = 2000;
+#endif
+
  pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg));
  if (pVnode->pWal == NULL) {
    vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno));
@ -194,4 +203,9 @@ void vnodeStop(SVnode *pVnode) {}

 int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }

-void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }
+void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) {
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = pVnode->state.committed;
+  pSnapshot->lastApplyTerm = pVnode->state.commitTerm;
+  pSnapshot->lastConfigIndex = -1;
+}
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@ -13,24 +13,27 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

-#include "vnodeInt.h"
+#include "vnd.h"

-struct SVSnapshotReader {
-  SVnode              *pVnode;
-  int64_t              sver;
-  int64_t              ever;
-  int8_t               isMetaEnd;
-  int8_t               isTsdbEnd;
-  SMetaSnapshotReader *pMetaReader;
-  STsdbSnapshotReader *pTsdbReader;
-  void                *pData;
-  int32_t              nData;
+// SVSnapReader ========================================================
+struct SVSnapReader {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+  // meta
+  int8_t           metaDone;
+  SMetaSnapReader *pMetaReader;
+  // tsdb
+  int8_t           tsdbDone;
+  STsdbSnapReader *pTsdbReader;
+  uint8_t         *pData;
 };

-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever) {
-  SVSnapshotReader *pReader = NULL;
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
+  int32_t       code = 0;
+  SVSnapReader *pReader = NULL;

-  pReader = (SVSnapshotReader *)taosMemoryCalloc(1, sizeof(*pReader));
+  pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
  if (pReader == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
@ -38,72 +41,169 @@ int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int
  pReader->pVnode = pVnode;
  pReader->sver = sver;
  pReader->ever = ever;
-  pReader->isMetaEnd = 0;
-  pReader->isTsdbEnd = 0;

-  if (metaSnapshotReaderOpen(pVnode->pMeta, &pReader->pMetaReader, sver, ever) < 0) {
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = metaSnapReaderOpen(pVnode->pMeta, sver, ever, &pReader->pMetaReader);
+  if (code) goto _err;

-  if (tsdbSnapshotReaderOpen(pVnode->pTsdb, &pReader->pTsdbReader, sver, ever) < 0) {
-    metaSnapshotReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = tsdbSnapReaderOpen(pVnode->pTsdb, sver, ever, &pReader->pTsdbReader);
+  if (code) goto _err;

-_exit:
  *ppReader = pReader;
-  return 0;
+  return code;

 _err:
+  vError("vgId:%d vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
  *ppReader = NULL;
-  return -1;
+  return code;
 }

-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader) {
-  if (pReader) {
-    vnodeFree(pReader->pData);
-    tsdbSnapshotReaderClose(pReader->pTsdbReader);
-    metaSnapshotReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-  }
-  return 0;
-}
-
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData) {
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
  int32_t code = 0;

-  if (!pReader->isMetaEnd) {
-    code = metaSnapshotRead(pReader->pMetaReader, &pReader->pData, &pReader->nData);
+  tFree(pReader->pData);
+  if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
+  if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
+  taosMemoryFree(pReader);
+
+  return code;
+}
+
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
+  int32_t code = 0;
+
+  if (!pReader->metaDone) {
+    code = metaSnapRead(pReader->pMetaReader, &pReader->pData);
    if (code) {
      if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isMetaEnd = 1;
+        pReader->metaDone = 1;
      } else {
-        return code;
+        goto _err;
      }
    } else {
      *ppData = pReader->pData;
-      *nData = pReader->nData;
-      return code;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
+      goto _exit;
    }
  }

-  if (!pReader->isTsdbEnd) {
-    code = tsdbSnapshotRead(pReader->pTsdbReader, &pReader->pData, &pReader->nData);
+  if (!pReader->tsdbDone) {
+    code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData);
    if (code) {
      if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isTsdbEnd = 1;
+        pReader->tsdbDone = 1;
      } else {
-        return code;
+        goto _err;
      }
    } else {
      *ppData = pReader->pData;
-      *nData = pReader->nData;
-      return code;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
+      goto _exit;
    }
  }

  code = TSDB_CODE_VND_READ_END;
+
+_exit:
+  return code;
+
+_err:
+  vError("vgId:% snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
+  return code;
+}
+
+// SVSnapWriter ========================================================
+struct SVSnapWriter {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+  // meta
+  SMetaSnapWriter *pMetaSnapWriter;
+  // tsdb
+  STsdbSnapWriter *pTsdbSnapWriter;
+};
+
+static int32_t vnodeSnapRollback(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t vnodeSnapCommit(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
+  int32_t       code = 0;
+  SVSnapWriter *pWriter = NULL;
+
+  // alloc
+  pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pVnode = pVnode;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
+  int32_t code = 0;
+
+  if (rollback) {
+    code = vnodeSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = vnodeSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+
+  taosMemoryFree(pWriter);
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshow writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
+  int32_t       code = 0;
+  SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData;
+  SVnode       *pVnode = pWriter->pVnode;
+
+  ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData);
+
+  if (pSnapDataHdr->type == 0) {
+    // meta
+    if (pWriter->pMetaSnapWriter == NULL) {
+      code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
+  } else {
+    // tsdb
+    if (pWriter->pTsdbSnapWriter == NULL) {
+      code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshot write failed since %s", TD_VID(pVnode), tstrerror(code));
  return code;
 }
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@ -24,7 +24,8 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
 static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);

@ -143,6 +144,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
         version);

  pVnode->state.applied = version;
+  pVnode->state.applyTerm = pMsg->info.conn.applyTerm;

  // skip header
  pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
@ -214,9 +216,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
      vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp);
      break;
    case TDMT_VND_ALTER_HASHRANGE:
-      vnodeProcessAlterHasnRangeReq(pVnode, version, pReq, len, pRsp);
+      vnodeProcessAlterHashRangeReq(pVnode, version, pReq, len, pRsp);
      break;
    case TDMT_VND_ALTER_CONFIG:
+      vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp);
      break;
    case TDMT_VND_COMMIT:
      goto _do_commit;
@ -885,7 +888,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void
  return 0;
 }

-static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
  vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode));

  // todo
@ -895,6 +898,18 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo
  return 0;
 }

+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  SAlterVnodeReq alterReq = {0};
+  if (tDeserializeSAlterVnodeReq(pReq, len, &alterReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return TSDB_CODE_INVALID_MSG;
+  }
+
+  vInfo("vgId:%d, start to alter vnode config, cacheLast:%d cacheLastSize:%d", TD_VID(pVnode), alterReq.cacheLast,
+        alterReq.cacheLastSize);
+  return 0;
+}
+
 static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
  int32_t     code = 0;
  SDecoder   *pCoder = &(SDecoder){0};
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@ -450,31 +450,82 @@ static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta
         syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType));
 }

+#define USE_TSDB_SNAPSHOT
+
 static int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void *pParam, void **ppReader) {
+#ifdef USE_TSDB_SNAPSHOT
  SVnode         *pVnode = pFsm->data;
  SSnapshotParam *pSnapshotParam = pParam;
-  int32_t         code =
-      vnodeSnapshotReaderOpen(pVnode, (SVSnapshotReader **)ppReader, pSnapshotParam->start, pSnapshotParam->end);
+  int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader);
  return code;
+#else
+  *ppReader = taosMemoryMalloc(32);
+  return 0;
+#endif
 }

 static int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) {
+#ifdef USE_TSDB_SNAPSHOT
  SVnode *pVnode = pFsm->data;
-  int32_t code = vnodeSnapshotReaderClose(pReader);
+  int32_t code = vnodeSnapReaderClose(pReader);
  return code;
+#else
+  taosMemoryFree(pReader);
+  return 0;
+#endif
 }

 static int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
+#ifdef USE_TSDB_SNAPSHOT
  SVnode *pVnode = pFsm->data;
-  int32_t code = vnodeSnapshotRead(pReader, (const void **)ppBuf, len);
+  int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
  return code;
+#else
+  static int32_t times = 0;
+  if (times++ < 5) {
+    *len = 64;
+    *ppBuf = taosMemoryMalloc(*len);
+    snprintf(*ppBuf, *len, "snapshot block %d", times);
+  } else {
+    *len = 0;
+    *ppBuf = NULL;
+  }
+  return 0;
+#endif
 }

-static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWriter) { return 0; }
+static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWriter) {
+#ifdef USE_TSDB_SNAPSHOT
+  SVnode         *pVnode = pFsm->data;
+  SSnapshotParam *pSnapshotParam = pParam;
+  int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter);
+  return code;
+#else
+  *ppWriter = taosMemoryMalloc(32);
+  return 0;
+#endif
+}

-static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { return 0; }
+static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) {
+#ifdef USE_TSDB_SNAPSHOT
+  SVnode *pVnode = pFsm->data;
+  int32_t code = vnodeSnapWriterClose(pWriter, !isApply);
+  return code;
+#else
+  taosMemoryFree(pWriter);
+  return 0;
+#endif
+}

-static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { return 0; }
+static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) {
+#ifdef USE_TSDB_SNAPSHOT
+  SVnode *pVnode = pFsm->data;
+  int32_t code = vnodeSnapWrite(pWriter, pBuf, len);
+  return code;
+#else
+  return 0;
+#endif
+}

 static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
  SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
@ -497,7 +548,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {

 int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
  SSyncInfo syncInfo = {
-      .snapshotStrategy = SYNC_STRATEGY_NO_SNAPSHOT,
+      .snapshotStrategy = SYNC_STRATEGY_WAL_FIRST,
+      //.snapshotStrategy = SYNC_STRATEGY_NO_SNAPSHOT,
      .batchSize = 10,
      .vgId = pVnode->config.vgId,
      .isStandBy = pVnode->config.standby,
--- a/source/libs/catalog/inc/catalogInt.h
+++ b/source/libs/catalog/inc/catalogInt.h
@ -166,7 +166,7 @@ typedef struct SCtgDBCache {
  int8_t           deleted;
  SCtgVgCache      vgCache;
  SHashObj        *tbCache;         // key:tbname, value:SCtgTbCache
-  SHashObj        *stbCache;        // key:suid, value:STableMeta*
+  SHashObj        *stbCache;        // key:suid, value:char*
 } SCtgDBCache;

 typedef struct SCtgRentSlot {
@ -480,37 +480,35 @@ typedef struct SCtgOperation {

 #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000

-#define CTG_IS_LOCKED(_lock) atomic_load_32((_lock))
-
 #define CTG_LOCK(type, _lock) do {   \
  if (CTG_READ == (type)) {          \
-    assert(atomic_load_32((_lock)) >= 0);  \
-    CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG RLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
    taosRLockLatch(_lock);           \
-    CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
-    assert(atomic_load_32((_lock)) > 0);  \
+    CTG_LOCK_DEBUG("CTG RLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) > 0);  \
  } else {                                                \
-    assert(atomic_load_32((_lock)) >= 0);  \
-    CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
+    assert(atomic_load_64((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG WLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__);  \
    taosWLockLatch(_lock);                                \
-    CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
-    assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
+    CTG_LOCK_DEBUG("CTG WLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__);  \
+    assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
  }                                                       \
 } while (0)

 #define CTG_UNLOCK(type, _lock) do {                       \
  if (CTG_READ == (type)) {                                \
-    assert(atomic_load_32((_lock)) > 0);  \
-    CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) > 0);  \
+    CTG_LOCK_DEBUG("CTG RULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
    taosRUnLockLatch(_lock);                              \
-    CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
-    assert(atomic_load_32((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG RULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) >= 0);  \
  } else {                                                \
-    assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
-    CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
+    CTG_LOCK_DEBUG("CTG WULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
    taosWUnLockLatch(_lock);                              \
-    CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
-    assert(atomic_load_32((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG WULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) >= 0);  \
  }                                                       \
 } while (0)

--- a/source/libs/catalog/src/ctgAsync.c
+++ b/source/libs/catalog/src/ctgAsync.c
@ -790,8 +790,12 @@ _return:
 int32_t ctgCallUserCb(void* param) {
  SCtgJob* pJob = (SCtgJob*)param;

+  qDebug("QID:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode));
+  
  (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode);

+  qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId);
+
  taosRemoveRef(gCtgMgmt.jobPool, pJob->refId);

  return TSDB_CODE_SUCCESS;
@ -822,8 +826,6 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {

 _return:

-  qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(code));
-
  pJob->jobResCode = code;

  //taosSsleep(2);
--- a/source/libs/catalog/test/catalogTests.cpp
+++ b/source/libs/catalog/test/catalogTests.cpp
@ -109,7 +109,7 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) {
  createReq.compression = 2;
  createReq.replications = 1;
  createReq.strict = 1;
-  createReq.cacheLastRow = 0;
+  createReq.cacheLast = 0;
  createReq.ignoreExist = 1;

  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/libs/command/src/command.c
+++ b/source/libs/command/src/command.c
@ -223,7 +223,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S
                 "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm "
                 "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
                 "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d",
-                 dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod,
+                 dbFName, pCfg->buffer, pCfg->cacheLast, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod,
                 pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages,
                 pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups,
                 1 == pCfg->numOfStables);
--- a/source/libs/executor/inc/executil.h
+++ b/source/libs/executor/inc/executil.h
@ -126,4 +126,6 @@ void    cleanupQueryTableDataCond(SQueryTableDataCond* pCond);

 int32_t convertFillType(int32_t mode);

+int32_t resultrowComparAsc(const void* p1, const void* p2);
+
 #endif  // TDENGINE_QUERYUTIL_H
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@ -144,6 +144,7 @@ typedef struct {
  void*          metaBlk;       // for tmq fetching meta
  SSDataBlock*   pullOverBlk;   // for streaming
  SWalFilterCond cond;
+  int64_t        lastScanUid;
 } SStreamTaskInfo;

 typedef struct SExecTaskInfo {
@ -278,9 +279,6 @@ typedef struct STableScanInfo {
  SScanInfo       scanInfo;
  int32_t         scanTimes;
  SNode*          pFilterNode;  // filter info, which is push down by optimizer
-  SqlFunctionCtx* pCtx;         // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup
-  int32_t*        rowEntryInfoOffset;  // todo: remove this by using SExprSup
-  SExprInfo*      pExpr;// todo: remove this by using SExprSup

  SSDataBlock*    pResBlock;
  SArray*         pColMatchInfo;
@ -289,19 +287,17 @@ typedef struct STableScanInfo {
  int32_t         scanFlag;     // table scan flag to denote if it is a repeat/reverse/main scan
  int32_t         dataBlockLoadFlag;
  SInterval       interval;     // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded.
-
  SSampleExecInfo sample;       // sample execution info
-  int32_t         curTWinIdx;

  int32_t         currentGroupId;
  int32_t         currentTable;
-  uint64_t        queryId;   // todo remove it
-  uint64_t        taskId;    // todo remove it

+#if 0
  struct {
    uint64_t uid;
    int64_t ts;
  } lastStatus;
+#endif

  int8_t scanMode;
  int8_t noTable;
@ -470,11 +466,22 @@ typedef struct SIntervalAggOperatorInfo {
  SNode *pCondition;
 } SIntervalAggOperatorInfo;

+typedef struct SMergeAlignedIntervalAggOperatorInfo {
+  SIntervalAggOperatorInfo *intervalAggOperatorInfo;
+
+  bool         hasGroupId;
+  uint64_t     groupId;
+  SSDataBlock* prefetchedBlock;
+  bool         inputBlocksFinished;
+
+  SNode*       pCondition;
+} SMergeAlignedIntervalAggOperatorInfo;
+
 typedef struct SStreamFinalIntervalOperatorInfo {
  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
  SOptrBasicInfo     binfo;              // basic info
  SAggSupporter      aggSup;             // aggregate supporter
-
+  SExprSupp          scalarSupp;         // supporter for perform scalar function
  SGroupResInfo      groupResInfo;       // multiple results build supporter
  SInterval          interval;           // interval info
  int32_t            primaryTsIndex;     // primary time stamp slot id from result of downstream operator.
@ -533,6 +540,9 @@ typedef struct SIndefOperatorInfo {
  SArray*            pPseudoColInfo;
  SExprSupp          scalarSup;
  SNode*             pCondition;
+  uint64_t           groupId;
+
+  SSDataBlock*       pNextGroupRes;
 } SIndefOperatorInfo;

 typedef struct SFillOperatorInfo {
@ -544,6 +554,8 @@ typedef struct SFillOperatorInfo {
  bool              multigroupResult;
  STimeWindow       win;
  SNode*            pCondition;
+  SArray*           pColMatchColInfo;
+  int32_t           primaryTsCol;
 } SFillOperatorInfo;

 typedef struct SGroupbyOperatorInfo {
@ -622,6 +634,7 @@ typedef struct SStateWindowInfo {
 typedef struct SStreamSessionAggOperatorInfo {
  SOptrBasicInfo       binfo;
  SStreamAggSupporter  streamAggSup;
+  SExprSupp            scalarSupp;         // supporter for perform scalar function
  SGroupResInfo        groupResInfo;
  int64_t              gap;             // session window gap
  int32_t              primaryTsIndex;  // primary timestamp slot id
@ -672,11 +685,12 @@ typedef struct SStateWindowOperatorInfo {
 typedef struct SStreamStateAggOperatorInfo {
  SOptrBasicInfo       binfo;
  SStreamAggSupporter  streamAggSup;
+  SExprSupp            scalarSupp;      // supporter for perform scalar function
  SGroupResInfo        groupResInfo;
  int32_t              primaryTsIndex;  // primary timestamp slot id
  int32_t              order;           // current SSDataBlock scan order
  STimeWindowAggSupp   twAggSup;
-  SColumn              stateCol;  // start row index
+  SColumn              stateCol;
  SqlFunctionCtx*      pDummyCtx;       // for combine
  SSDataBlock*         pDelRes;
  SHashObj*            pSeDeleted;
@ -765,6 +779,8 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
                                  int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
                                  SArray* pColList);
 void    getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win);
+STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order);
+
 int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag);
 int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);

@ -902,7 +918,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result);
 int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length);

 STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
-                                int32_t precision, int32_t order);
+                                int32_t order);
 int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
    __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order);
 int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
@ -918,7 +934,6 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs,
    TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted);
 bool functionNeedToExecute(SqlFunctionCtx* pCtx);

-int32_t compareTimeWindow(const void* p1, const void* p2, const void* param);
 int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition,
                                       SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,
                                       SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@ -13,6 +13,7 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include <common/ttime.h>
 #include "function.h"
 #include "functionMgt.h"
 #include "index.h"
@ -76,7 +77,7 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
  pGroupResInfo->index = 0;
 }

-static int32_t resultrowComparAsc(const void* p1, const void* p2) {
+int32_t resultrowComparAsc(const void* p1, const void* p2) {
  SResKeyPos* pp1 = *(SResKeyPos**)p1;
  SResKeyPos* pp2 = *(SResKeyPos**)p2;

@ -769,12 +770,9 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi

  // pCond->twindow = pTableScanNode->scanRange;
  // TODO: get it from stable scan node
-  pCond->numOfTWindows = 1;
-  pCond->twindows = taosMemoryCalloc(pCond->numOfTWindows, sizeof(STimeWindow));
-  pCond->twindows[0] = pTableScanNode->scanRange;
-  pCond->suid = pTableScanNode->scan.suid;
-
-  pCond->type = BLOCK_LOAD_OFFSET_ORDER;
+  pCond->twindows = pTableScanNode->scanRange;
+  pCond->suid     = pTableScanNode->scan.suid;
+  pCond->type     = BLOCK_LOAD_OFFSET_ORDER;
  pCond->startVersion = -1;
  pCond->endVersion   = -1;
  //  pCond->type = pTableScanNode->scanFlag;
@ -826,3 +824,87 @@ int32_t convertFillType(int32_t mode) {

  return type;
 }
+
+static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) {
+  if (ascQuery) {
+    getAlignQueryTimeWindow(pInterval, pInterval->precision, ts, w);
+  } else {
+    // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
+    getAlignQueryTimeWindow(pInterval, pInterval->precision, ts, w);
+
+    int64_t key = w->skey;
+    while (key < ts) {  // moving towards end
+      key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
+      if (key >= ts) {
+        break;
+      }
+
+      w->skey = key;
+    }
+  }
+}
+
+static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) {
+  STimeWindow  w =  {0};
+
+  if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
+    w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision);
+    w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
+  } else {
+    int64_t st = w.skey;
+
+    if (st > ts) {
+      st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
+    }
+
+    int64_t et = st + pInterval->interval - 1;
+    if (et < ts) {
+      st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
+    }
+
+    w.skey = st;
+    w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
+  }
+
+  return w;
+}
+
+STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) {
+  int32_t factor = (order == TSDB_ORDER_ASC)? -1:1;
+
+  STimeWindow win = *pWindow;
+  STimeWindow save = win;
+  while(win.skey <= ts && win.ekey >= ts) {
+    save = win;
+    win.skey = taosTimeAdd(win.skey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
+    win.ekey = taosTimeAdd(win.ekey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
+  }
+
+  return save;
+}
+
+// get the correct time window according to the handled timestamp
+STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
+                                int32_t order) {
+  STimeWindow w = {0};
+  if (pResultRowInfo->cur.pageId == -1) {  // the first window, from the previous stored value
+    getInitialStartTimeWindow(pInterval, ts, &w, (order == TSDB_ORDER_ASC));
+    w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
+    return w;
+  }
+
+  w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win;
+
+  // in case of typical time window, we can calculate time window directly.
+  if (w.skey > ts || w.ekey < ts) {
+    w = doCalculateTimeWindow(ts, pInterval);
+  }
+
+  if (pInterval->interval != pInterval->sliding) {
+    // it is an sliding window query, in which sliding value is not equalled to
+    // interval value, and we need to find the first qualified time window.
+    w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order);
+  }
+
+  return w;
+}
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@ -60,9 +60,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
        taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
        taosArrayPush(pInfo->pBlockLists, &p);
      }
-    } else if (type == STREAM_INPUT__TABLE_SCAN) {
-      // do nothing
-      ASSERT(pInfo->blockType == STREAM_INPUT__TABLE_SCAN);
+      /*} else if (type == STREAM_INPUT__TABLE_SCAN) {*/
+      /*ASSERT(pInfo->blockType == STREAM_INPUT__TABLE_SCAN);*/
    } else {
      ASSERT(0);
    }
@ -71,6 +70,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
  }
 }

+#if 0
 int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) {
  if (tinfo == NULL) {
    return TSDB_CODE_QRY_APP_ERROR;
@ -78,6 +78,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) {
  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
  return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__TABLE_SCAN, 0, NULL);
 }
+#endif

 int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) {
  return qSetMultiStreamInput(tinfo, input, 1, type, assignUid);
@ -184,7 +185,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
  return code;
 }

-int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
+int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
                                      int32_t* tversion) {
  ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL);
  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
--- a/source/libs/executor/src/executorMain.c
+++ b/source/libs/executor/src/executorMain.c
@ -280,7 +280,7 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
  return 0;
 }

-int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
  SOperatorInfo* pOperator = pTaskInfo->pRoot;
  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
@ -293,8 +293,52 @@ int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
    pOperator->status = OP_OPENED;
    if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
      SStreamScanInfo* pInfo = pOperator->info;
-      if (tqSeekVer(pInfo->tqReader, pOffset->version) < 0) {
-        return -1;
+      if (pOffset->type == TMQ_OFFSET__LOG) {
+        if (tqSeekVer(pInfo->tqReader, pOffset->version) < 0) {
+          return -1;
+        }
+        ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version);
+      } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+        /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/
+        int64_t uid = pOffset->uid;
+        int64_t ts = pOffset->ts;
+
+        if (uid == 0) {
+          if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) {
+            STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
+            uid = pTableInfo->uid;
+            ts = INT64_MIN;
+          }
+        }
+        if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||
+            pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {
+          STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
+          int32_t         tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
+          bool            found = false;
+          for (int32_t i = 0; i < tableSz; i++) {
+            STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
+            if (pTableInfo->uid == uid) {
+              found = true;
+              pTableScanInfo->currentTable = i;
+            }
+          }
+
+          // TODO after dropping table, table may be not found
+          ASSERT(found);
+
+          tsdbSetTableId(pTableScanInfo->dataReader, uid);
+          int64_t oldSkey = pTableScanInfo->cond.twindows.skey;
+          pTableScanInfo->cond.twindows.skey = ts + 1;
+          tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
+          pTableScanInfo->cond.twindows.skey = oldSkey;
+          pTableScanInfo->scanTimes = 0;
+
+          qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts,
+                 pTableScanInfo->currentTable, tableSz);
+        }
+
+      } else {
+        ASSERT(0);
      }
      return 0;
    } else {
@ -306,6 +350,7 @@ int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
  return 0;
 }

+#if 0
 int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) {
  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;

@ -325,3 +370,4 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {

  return doGetScanStatus(pTaskInfo->pRoot, uid, ts);
 }
+#endif
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@ -13,6 +13,7 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

+#include <executorimpl.h>
 #include "filter.h"
 #include "function.h"
 #include "functionMgt.h"
@ -544,9 +545,7 @@ static int32_t doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunct
      if (pCtx[k].fpSet.process == NULL) {
        continue;
      }
-#ifdef BUF_PAGE_DEBUG
-      qDebug("page_process");
-#endif
+
      int32_t code = pCtx[k].fpSet.process(&pCtx[k]);
      if (code != TSDB_CODE_SUCCESS) {
        qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code));
@ -570,8 +569,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
  setPseudoOutputColInfo(pResult, pCtx, pPseudoList);
  pResult->info.groupId = pSrcBlock->info.groupId;

-  // if the source equals to the destination, it is to create a new column as the result of scalar function or some
-  // operators.
+  // if the source equals to the destination, it is to create a new column as the result of scalar
+  // function or some operators.
  bool createNewColModel = (pResult == pSrcBlock);

  int32_t numOfRows = 0;
@ -579,17 +578,17 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
  for (int32_t k = 0; k < numOfOutput; ++k) {
    int32_t         outputSlotId = pExpr[k].base.resSchema.slotId;
    SqlFunctionCtx* pfCtx = &pCtx[k];
+    SInputColumnInfoData* pInputData = &pfCtx->input;

    if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) {  // it is a project query
      SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
      if (pResult->info.rows > 0 && !createNewColModel) {
-        colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0],
-                        pfCtx->input.numOfRows);
+        colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pInputData->pData[0], pInputData->numOfRows);
      } else {
-        colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info);
+        colDataAssign(pColInfoData, pInputData->pData[0], pInputData->numOfRows, &pResult->info);
      }

-      numOfRows = pfCtx->input.numOfRows;
+      numOfRows = pInputData->numOfRows;
    } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) {
      SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);

@ -622,14 +621,12 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
      numOfRows = dest.numOfRows;
      taosArrayDestroy(pBlockList);
    } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) {
-      ASSERT(!fmIsAggFunc(pfCtx->functionId));
-
      // _rowts/_c0, not tbname column
      if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) {
        // do nothing
      } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) {
-        SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]);
-        pfCtx->fpSet.init(&pCtx[k], pResInfo);
+        SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx);
+        pfCtx->fpSet.init(pfCtx, pResInfo);

        pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
        pfCtx->offset = createNewColModel ? 0 : pResult->info.rows;  // set the start offset
@ -641,6 +638,23 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
        }

        numOfRows = pfCtx->fpSet.process(pfCtx);
+      } else if (fmIsAggFunc(pfCtx->functionId)) {
+        // _group_key function for "partition by tbname" + csum(col_name) query
+        SColumnInfoData* pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
+        int32_t slotId = pfCtx->param[0].pCol->slotId;
+
+        // todo handle the json tag
+        SColumnInfoData* pInput = taosArrayGet(pSrcBlock->pDataBlock, slotId);
+        for(int32_t f = 0; f < pSrcBlock->info.rows; ++f) {
+          bool isNull = colDataIsNull_s(pInput, f);
+          if (isNull) {
+            colDataAppendNULL(pOutput, pResult->info.rows + f);
+          } else {
+            char* data = colDataGetData(pInput, f);
+            colDataAppend(pOutput, pResult->info.rows + f, data, isNull);
+          }
+        }
+
      } else {
        SArray* pBlockList = taosArrayInit(4, POINTER_BYTES);
        taosArrayPush(pBlockList, &pSrcBlock);
@ -674,25 +688,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
  return TSDB_CODE_SUCCESS;
 }

-static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) {
-  if (IS_VAR_DATA_TYPE(type)) {
-    // todo disable this
-
-    //    if (pResultRow->key == NULL) {
-    //      pResultRow->key = taosMemoryMalloc(varDataTLen(pData));
-    //      varDataCopy(pResultRow->key, pData);
-    //    } else {
-    //      ASSERT(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0);
-    //    }
-  } else {
-    int64_t v = -1;
-    GET_TYPED_DATA(v, int64_t, type, pData);
-
-    pResultRow->win.skey = v;
-    pResultRow->win.ekey = v;
-  }
-}
-
 bool functionNeedToExecute(SqlFunctionCtx* pCtx) {
  struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);

@ -1038,6 +1033,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI
 #endif

 static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
+#if 0
  SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
  uint32_t        status = BLK_DATA_NOT_LOAD;

@ -1059,6 +1055,8 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData
  }

  return status;
+#endif
+  return 0;
 }

 int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
@ -2844,7 +2842,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
    }
  }
 }
-
+#if 0
 int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
  uint8_t type = pOperator->operatorType;

@ -2887,12 +2885,11 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
      ASSERT(found);

      tsdbSetTableId(pInfo->dataReader, uid);
-      int64_t oldSkey = pInfo->cond.twindows[0].skey;
-      pInfo->cond.twindows[0].skey = ts + 1;
-      tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0);
-      pInfo->cond.twindows[0].skey = oldSkey;
+      int64_t oldSkey = pInfo->cond.twindows.skey;
+      pInfo->cond.twindows.skey = ts + 1;
+      tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
+      pInfo->cond.twindows.skey = oldSkey;
      pInfo->scanTimes = 0;
-      pInfo->curTWinIdx = 0;

      qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, ts,
             pInfo->currentTable, tableSz);
@ -2930,6 +2927,7 @@ int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {

  return TSDB_CODE_SUCCESS;
 }
+#endif

 // this is a blocking operator
 static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
@ -3341,8 +3339,8 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResult
                                               SExecTaskInfo* pTaskInfo) {
  pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;

-  int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pInfo->win.ekey
-                                                                   : pInfo->existNewGroupBlock->info.window.ekey;
+  int64_t ekey =
+      Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pInfo->win.ekey : pInfo->existNewGroupBlock->info.window.ekey;
  taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));

  taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey);
@ -3821,6 +3819,40 @@ _error:
  return NULL;
 }

+static void doHandleDataBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, SOperatorInfo* downstream, SExecTaskInfo* pTaskInfo) {
+  int32_t order = 0;
+  int32_t scanFlag = 0;
+
+  SIndefOperatorInfo* pIndefInfo = pOperator->info;
+  SOptrBasicInfo*     pInfo = &pIndefInfo->binfo;
+  SExprSupp*          pSup = &pOperator->exprSupp;
+
+  // the pDataBlock are always the same one, no need to call this again
+  int32_t code = getTableScanInfo(downstream, &order, &scanFlag);
+  if (code != TSDB_CODE_SUCCESS) {
+    longjmp(pTaskInfo->env, code);
+  }
+
+  // there is an scalar expression that needs to be calculated before apply the group aggregation.
+  SExprSupp* pScalarSup = &pIndefInfo->scalarSup;
+  if (pScalarSup->pExprInfo != NULL) {
+    code = projectApplyFunctions(pScalarSup->pExprInfo, pBlock, pBlock, pScalarSup->pCtx, pScalarSup->numOfExprs,
+                                 pIndefInfo->pPseudoColInfo);
+    if (code != TSDB_CODE_SUCCESS) {
+      longjmp(pTaskInfo->env, code);
+    }
+  }
+
+  setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false);
+  blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
+
+  code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs,
+                               pIndefInfo->pPseudoColInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    longjmp(pTaskInfo->env, code);
+  }
+}
+
 static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
  SIndefOperatorInfo* pIndefInfo = pOperator->info;
  SOptrBasicInfo*     pInfo = &pIndefInfo->binfo;
@ -3835,8 +3867,6 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
  }

  int64_t st = 0;
-  int32_t order = 0;
-  int32_t scanFlag = 0;

  if (pOperator->cost.openCost == 0) {
    st = taosGetTimestampUs();
@ -3844,42 +3874,54 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {

  SOperatorInfo* downstream = pOperator->pDownstream[0];

-  while (1) {
-    // The downstream exec may change the value of the newgroup, so use a local variable instead.
-    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
-    if (pBlock == NULL) {
-      doSetOperatorCompleted(pOperator);
-      break;
+  while(1) {
+    // here we need to handle the existsed group results
+    if (pIndefInfo->pNextGroupRes != NULL) { // todo extract method
+      for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
+        SqlFunctionCtx* pCtx = &pSup->pCtx[k];
+
+        SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+        pResInfo->initialized = false;
+        pCtx->pOutput = NULL;
+      }
+
+      doHandleDataBlock(pOperator, pIndefInfo->pNextGroupRes, downstream, pTaskInfo);
+      pIndefInfo->pNextGroupRes = NULL;
    }

-    // the pDataBlock are always the same one, no need to call this again
-    int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
-    if (code != TSDB_CODE_SUCCESS) {
-      longjmp(pTaskInfo->env, code);
-    }
+    if (pInfo->pRes->info.rows < pOperator->resultInfo.threshold) {
+      while (1) {
+        // The downstream exec may change the value of the newgroup, so use a local variable instead.
+        SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+        if (pBlock == NULL) {
+          doSetOperatorCompleted(pOperator);
+          break;
+        }

-    // there is an scalar expression that needs to be calculated before apply the group aggregation.
-    SExprSupp* pScalarSup = &pIndefInfo->scalarSup;
-    if (pScalarSup->pExprInfo != NULL) {
-      code = projectApplyFunctions(pScalarSup->pExprInfo, pBlock, pBlock, pScalarSup->pCtx, pScalarSup->numOfExprs,
-                                   pIndefInfo->pPseudoColInfo);
-      if (code != TSDB_CODE_SUCCESS) {
-        longjmp(pTaskInfo->env, code);
+        if (pIndefInfo->groupId == 0 && pBlock->info.groupId != 0) {
+          pIndefInfo->groupId = pBlock->info.groupId;  // this is the initial group result
+        } else {
+          if (pIndefInfo->groupId != pBlock->info.groupId) {  // reset output buffer and computing status
+            pIndefInfo->groupId = pBlock->info.groupId;
+            pIndefInfo->pNextGroupRes = pBlock;
+            break;
+          }
+        }
+
+        doHandleDataBlock(pOperator, pBlock, downstream, pTaskInfo);
+        if (pInfo->pRes->info.rows >= pOperator->resultInfo.threshold) {
+          break;
+        }
      }
    }

-    setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false);
-    blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
-
-    code = projectApplyFunctions(pOperator->exprSupp.pExprInfo, pInfo->pRes, pBlock, pSup->pCtx,
-                                 pOperator->exprSupp.numOfExprs, pIndefInfo->pPseudoColInfo);
-    if (code != TSDB_CODE_SUCCESS) {
-      longjmp(pTaskInfo->env, code);
+    doFilter(pIndefInfo->pCondition, pInfo->pRes);
+    size_t rows = pInfo->pRes->info.rows;
+    if (rows >= 0) {
+      break;
    }
  }

-  doFilter(pIndefInfo->pCondition, pInfo->pRes);
-
  size_t rows = pInfo->pRes->info.rows;
  pOperator->resultInfo.totalRows += rows;

@ -3924,24 +3966,23 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
  if (numOfRows * pResBlock->info.rowSize > TWOMB) {
    numOfRows = TWOMB / pResBlock->info.rowSize;
  }
+
  initResultSizeInfo(pOperator, numOfRows);

-  initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
+  initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
  initBasicInfo(&pInfo->binfo, pResBlock);

  setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr);

-  pInfo->binfo.pRes = pResBlock;
-  pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr);
-  pInfo->pCondition = pPhyNode->node.pConditions;
+  pInfo->binfo.pRes    = pResBlock;
+  pInfo->pCondition    = pPhyNode->node.pConditions;
+  pInfo->pPseudoColInfo= setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr);

-  pOperator->name = "IndefinitOperator";
-  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT;
-  pOperator->blocking = false;
-  pOperator->status = OP_NOT_OPENED;
-  pOperator->info = pInfo;
-  pOperator->exprSupp.pExprInfo = pExprInfo;
-  pOperator->exprSupp.numOfExprs = numOfExpr;
+  pOperator->name      = "IndefinitOperator";
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC;
+  pOperator->blocking  = false;
+  pOperator->status    = OP_NOT_OPENED;
+  pOperator->info      = pInfo;
  pOperator->pTaskInfo = pTaskInfo;

  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL,
@ -3967,12 +4008,13 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t

  STimeWindow w = TSWINDOW_INITIALIZER;
  getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey, &w);
+  w = getFirstQualifiedTimeWindow(win.skey, &w, pInterval, TSDB_ORDER_ASC);

  int32_t order = TSDB_ORDER_ASC;
  pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id);

  pInfo->win = win;
-  pInfo->p   = taosMemoryCalloc(numOfCols, POINTER_BYTES);
+  pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES);
  if (pInfo->pFillInfo == NULL || pInfo->p == NULL) {
    taosMemoryFree(pInfo->pFillInfo);
    taosMemoryFree(pInfo->p);
@ -3993,11 +4035,20 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
  int32_t      num = 0;
  SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
  SExprInfo*   pExprInfo = createExprInfo(pPhyFillNode->pTargets, NULL, &num);
-  SInterval*   pInterval = &((SIntervalAggOperatorInfo*)downstream->info)->interval;
-  int32_t      type = convertFillType(pPhyFillNode->mode);
+  SInterval*   pInterval =
+      QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
+            ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
+            : &((SIntervalAggOperatorInfo*)downstream->info)->interval;
+
+  int32_t type = convertFillType(pPhyFillNode->mode);

  SResultInfo* pResultInfo = &pOperator->resultInfo;
  initResultSizeInfo(pOperator, 4096);
+  pInfo->primaryTsCol = ((SColumnNode*)pPhyFillNode->pWStartTs)->slotId;
+
+  int32_t numOfOutputCols = 0;
+  SArray* pColMatchColInfo =
+      extractColMatchInfo(pPhyFillNode->pTargets, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);

  int32_t code = initFillInfo(pInfo, pExprInfo, num, (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange,
                              pResultInfo->capacity, pTaskInfo->id.str, pInterval, type);
@ -4005,17 +4056,18 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
    goto _error;
  }

-  pInfo->pRes = pResBlock;
-  pInfo->multigroupResult = multigroupResult;
-  pInfo->pCondition = pPhyFillNode->node.pConditions;
-  pOperator->name = "FillOperator";
-  pOperator->blocking = false;
-  pOperator->status = OP_NOT_OPENED;
-  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL;
-  pOperator->exprSupp.pExprInfo = pExprInfo;
+  pInfo->pRes                    = pResBlock;
+  pInfo->multigroupResult        = multigroupResult;
+  pInfo->pCondition              = pPhyFillNode->node.pConditions;
+  pInfo->pColMatchColInfo        = pColMatchColInfo;
+  pOperator->name                = "FillOperator";
+  pOperator->blocking            = false;
+  pOperator->status              = OP_NOT_OPENED;
+  pOperator->operatorType        = QUERY_NODE_PHYSICAL_PLAN_FILL;
+  pOperator->exprSupp.pExprInfo  = pExprInfo;
  pOperator->exprSupp.numOfExprs = num;
-  pOperator->info = pInfo;
-  pOperator->pTaskInfo = pTaskInfo;
+  pOperator->info                = pInfo;
+  pOperator->pTaskInfo           = pTaskInfo;

  pOperator->fpSet =
      createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo, NULL, NULL, NULL);
@ -4050,7 +4102,7 @@ static STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SRea

 static SArray* extractColumnInfo(SNodeList* pNodeList);

-int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) {
+int32_t extractTableSchemaInfo(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) {
  SMetaReader mr = {0};
  metaReaderInit(&mr, pHandle->meta, 0);
  int32_t code = metaGetTableEntryByUid(&mr, uid);
@ -4074,10 +4126,20 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI
  }

  metaReaderClear(&mr);
-
  return TSDB_CODE_SUCCESS;
 }

+static void cleanupTableSchemaInfo(SExecTaskInfo* pTaskInfo) {
+  taosMemoryFreeClear(pTaskInfo->schemaVer.dbname);
+  if (pTaskInfo->schemaVer.sw == NULL) {
+    return;
+  }
+
+  taosMemoryFree(pTaskInfo->schemaVer.sw->pSchema);
+  taosMemoryFree(pTaskInfo->schemaVer.sw);
+  taosMemoryFree(pTaskInfo->schemaVer.tablename);
+}
+
 static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum) {
  taosArrayClear(pTableListInfo->pGroupList);
  SArray* sortSupport = taosArrayInit(groupNum, sizeof(uint64_t));
@ -4251,7 +4313,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
        return NULL;
      }

-      code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
+      code = extractTableSchemaInfo(pHandle, pTableScanNode->scan.uid, pTaskInfo);
      if (code) {
        pTaskInfo->code = terrno;
        return NULL;
@ -4269,7 +4331,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
        pTaskInfo->code = code;
        return NULL;
      }
-      code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
+      code = extractTableSchemaInfo(pHandle, pTableScanNode->scan.uid, pTaskInfo);
      if (code) {
        pTaskInfo->code = terrno;
        return NULL;
@ -4347,9 +4409,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
        cond.colList->type = TSDB_DATA_TYPE_TIMESTAMP;
        cond.colList->bytes = sizeof(TSKEY);

-        cond.numOfTWindows = 1;
-        cond.twindows = taosMemoryCalloc(1, sizeof(STimeWindow));
-        cond.twindows[0] = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
+        cond.twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
        cond.suid = pBlockNode->suid;
        cond.type = BLOCK_LOAD_OFFSET_ORDER;
      }
@ -4368,7 +4428,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
      //        return NULL;
      //      }

-      int32_t code = extractTableSchemaVersion(pHandle, pScanNode->uid, pTaskInfo);
+      int32_t code = extractTableSchemaInfo(pHandle, pScanNode->uid, pTaskInfo);
      if (code != TSDB_CODE_SUCCESS) {
        pTaskInfo->code = code;
        return NULL;
@ -4465,7 +4525,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
                          .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};

    int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
-    pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pPhyNode->pConditions, pTaskInfo);
+    pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId,
+                                                   pPhyNode->pConditions, pTaskInfo);
  } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) {
    SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode;

@ -4504,8 +4565,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
    SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
    int32_t      tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;

-    pOptr =
-        createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pPhyNode->pConditions, pTaskInfo);
+    pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as,
+                                         pPhyNode->pConditions, pTaskInfo);
  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) {
    pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo);
  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) {
@ -4527,7 +4588,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo

    SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
    SColumn      col = extractColumnFromColumnNode(pColNode);
-    pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pPhyNode->pConditions, pTaskInfo);
+    pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pPhyNode->pConditions,
+                                          pTaskInfo);
  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) {
    pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo);
  } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) {
@ -4546,18 +4608,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
  return pOptr;
 }

-int32_t compareTimeWindow(const void* p1, const void* p2, const void* param) {
-  const SQueryTableDataCond* pCond = param;
-  const STimeWindow*         pWin1 = p1;
-  const STimeWindow*         pWin2 = p2;
-  if (pCond->order == TSDB_ORDER_ASC) {
-    return pWin1->skey - pWin2->skey;
-  } else if (pCond->order == TSDB_ORDER_DESC) {
-    return pWin2->skey - pWin1->skey;
-  }
-  return 0;
-}
-
 SArray* extractColumnInfo(SNodeList* pNodeList) {
  size_t  numOfCols = LIST_LENGTH(pNodeList);
  SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn));
@ -4879,11 +4929,8 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) {

  doDestroyTableList(&pTaskInfo->tableqinfoList);
  destroyOperatorInfo(pTaskInfo->pRoot);
-  //  taosArrayDestroy(pTaskInfo->summary.queryProfEvents);
-  //  taosHashCleanup(pTaskInfo->summary.operatorProfResults);
+  cleanupTableSchemaInfo(pTaskInfo);

-  taosMemoryFree(pTaskInfo->schemaVer.dbname);
-  taosMemoryFree(pTaskInfo->schemaVer.tablename);
  taosMemoryFreeClear(pTaskInfo->sql);
  taosMemoryFreeClear(pTaskInfo->id.str);
  taosMemoryFreeClear(pTaskInfo);
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@ -637,6 +637,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
  int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex);
  void* page = getBufPage(pInfo->pBuf, *pageId);

+  blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity);
  blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity);

  pInfo->pageIndex += 1;
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@ -40,8 +40,8 @@ static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta*
                                     const char* dbName);

 static int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
-                               SSDataBlock* pBlock, const char* idStr);
-static bool processBlockWithProbability(const SSampleExecInfo* pInfo);
+                                      SSDataBlock* pBlock, const char* idStr);
+static bool    processBlockWithProbability(const SSampleExecInfo* pInfo);

 bool processBlockWithProbability(const SSampleExecInfo* pInfo) {
 #if 0
@ -265,7 +265,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
  if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
    SExprSupp* pSup = &pTableScanInfo->pseudoSup;

-    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, GET_TASKID(pTaskInfo));
+    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock,
+                                          GET_TASKID(pTaskInfo));
    if (code != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, code);
    }
@ -293,17 +294,12 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction
  //  setupQueryRangeForReverseScan(pTableScanInfo);

  pTableScanInfo->cond.order = TSDB_ORDER_DESC;
-  for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
-    STimeWindow* pTWindow = &pTableScanInfo->cond.twindows[i];
-    TSWAP(pTWindow->skey, pTWindow->ekey);
-  }
-
-  SQueryTableDataCond* pCond = &pTableScanInfo->cond;
-  taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow);
+  STimeWindow* pTWindow = &pTableScanInfo->cond.twindows;
+  TSWAP(pTWindow->skey, pTWindow->ekey);
 }

 int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
-                            SSDataBlock* pBlock, const char* idStr) {
+                               SSDataBlock* pBlock, const char* idStr) {
  // currently only the tbname pseudo column
  if (numOfPseudoExpr == 0) {
    return TSDB_CODE_SUCCESS;
@ -313,7 +309,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int
  metaReaderInit(&mr, pHandle->meta, 0);
  int32_t code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
  if (code != TSDB_CODE_SUCCESS) {
-    qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr);
+    qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr);
    metaReaderClear(&mr);
    return terrno;
  }
@ -426,8 +422,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
    pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime;

    // todo refactor
-    pTableScanInfo->lastStatus.uid = pBlock->info.uid;
-    pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;
+    /*pTableScanInfo->lastStatus.uid = pBlock->info.uid;*/
+    /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/
+    pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA;
+    pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.uid;
+    pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;

    ASSERT(pBlock->info.uid != 0);
    return pBlock;
@ -446,16 +445,10 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {

  // do the ascending order traverse in the first place.
  while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) {
-    while (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) {
-      SSDataBlock* p = doTableScanImpl(pOperator);
-      if (p != NULL) {
-        ASSERT(p->info.uid != 0);
-        return p;
-      }
-      pTableScanInfo->curTWinIdx += 1;
-      if (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) {
-        tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx);
-      }
+    SSDataBlock* p = doTableScanImpl(pOperator);
+    if (p != NULL) {
+      ASSERT(p->info.uid != 0);
+      return p;
    }

    pTableScanInfo->scanTimes += 1;
@ -464,40 +457,25 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
      setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
      pTableScanInfo->scanFlag = REPEAT_SCAN;
      qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
-      for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
-        STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
-        qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
-      }
+
      // do prepare for the next round table scan operation
-      tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
-      pTableScanInfo->curTWinIdx = 0;
+      tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
    }
  }

  int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc;
  if (pTableScanInfo->scanTimes < total) {
    if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) {
-      prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, 0);
-      tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
-      pTableScanInfo->curTWinIdx = 0;
+      prepareForDescendingScan(pTableScanInfo, pOperator->exprSupp.pCtx, 0);
+      tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
    }

    qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
-    for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
-      STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
-      qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
-    }

    while (pTableScanInfo->scanTimes < total) {
-      while (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) {
-        SSDataBlock* p = doTableScanImpl(pOperator);
-        if (p != NULL) {
-          return p;
-        }
-        pTableScanInfo->curTWinIdx += 1;
-        if (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) {
-          tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx);
-        }
+      SSDataBlock* p = doTableScanImpl(pOperator);
+      if (p != NULL) {
+        return p;
      }

      pTableScanInfo->scanTimes += 1;
@ -508,12 +486,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {

        qDebug("%s start to repeat descending order scan data blocks due to query func required",
               GET_TASKID(pTaskInfo));
-        for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
-          STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
-          qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
-        }
-        tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
-        pTableScanInfo->curTWinIdx = 0;
+        tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
      }
    }
  }
@ -540,9 +513,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
      }
      STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable);
      tsdbSetTableId(pInfo->dataReader, pTableInfo->uid);
-      tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0);
+      tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
      pInfo->scanTimes = 0;
-      pInfo->curTWinIdx = 0;
    }
  }

@ -574,8 +546,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
  SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
  //  tsdbSetTableList(pInfo->dataReader, tableList);

-  tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0);
-  pInfo->curTWinIdx = 0;
+  tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
  pInfo->scanTimes = 0;

  result = doTableScanGroup(pOperator);
@ -646,7 +617,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
  pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
  pInfo->scanFlag = MAIN_SCAN;
  pInfo->pColMatchInfo = pColList;
-  pInfo->curTWinIdx = 0;
  pInfo->currentGroupId = -1;

  pOperator->name = "TableScanOperator";  // for debug purpose
@ -697,7 +667,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
  metaReaderInit(&mr, pMeta, 0);
  int32_t code = metaGetTableEntryByUid(&mr, uid);
  if (code != TSDB_CODE_SUCCESS) {
-    qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr);
+    qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr);
    metaReaderClear(&mr);
    return terrno;
  }
@ -711,7 +681,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
    uint64_t suid = mr.me.ctbEntry.suid;
    code = metaGetTableEntryByUid(&mr, suid);
    if (code != TSDB_CODE_SUCCESS) {
-      qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr);
+      qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr);
      metaReaderClear(&mr);
      return terrno;
    }
@ -738,12 +708,13 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
  }

  SBlockDistInfo* pBlockScanInfo = pOperator->info;
-  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

  STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
-  int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, &blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
+  int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, &blockDistInfo.rowSize,
+                                   GET_TASKID(pTaskInfo));
  if (code != TSDB_CODE_SUCCESS) {
-   longjmp(pTaskInfo->env, code);
+    longjmp(pTaskInfo->env, code);
  }

  tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
@ -875,12 +846,7 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou
 }

 void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
-  pTableScanInfo->cond.twindows[0] = *pWin;
-  pTableScanInfo->curTWinIdx = 0;
-  //  tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
-  // if (!pTableScanInfo->dataReader) {
-  //   return false;
-  // }
+  pTableScanInfo->cond.twindows = *pWin;
  pTableScanInfo->scanTimes = 0;
  pTableScanInfo->currentGroupId = -1;
 }
@ -938,7 +904,7 @@ static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t t
      setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
      (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL);
    } else {
-      win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, TSDB_ORDER_ASC);
+      win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, TSDB_ORDER_ASC);
      setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
      (*pRowIndex) +=
          getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
@ -1157,6 +1123,7 @@ static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32
    uidCol[i] = getGroupId(pOperator, uidCol[i]);
  }
 }
+
 static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock) {
  SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
  SOperatorInfo*  pOperator = pInfo->pStreamScanOp;
@ -1167,12 +1134,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
  pInfo->pRes->info.type = STREAM_NORMAL;
  pInfo->pRes->info.capacity = pBlock->info.rows;

-  // for generating rollup SMA result, each time is an independent time serie.
-  // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
-  if (pInfo->assignBlockUid) {
-    pInfo->pRes->info.groupId = pBlock->info.uid;
-  }
-
  uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
  if (groupIdPre) {
    pInfo->pRes->info.groupId = *groupIdPre;
@ -1180,6 +1141,12 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
    pInfo->pRes->info.groupId = 0;
  }

+  // for generating rollup SMA result, each time is an independent time serie.
+  // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
+  if (pInfo->assignBlockUid) {
+    pInfo->pRes->info.groupId = pBlock->info.uid;
+  }
+
  // todo extract method
  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
    SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
@ -1219,7 +1186,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock

  // currently only the tbname pseudo column
  if (pInfo->numOfPseudoExpr > 0) {
-    int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo));
+    int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes,
+                                          GET_TASKID(pTaskInfo));
    if (code != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, code);
    }
@ -1227,9 +1195,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock

  doFilter(pInfo->pCondition, pInfo->pRes);
  blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
-  if (pBlockInfo->rows > 0) {
-    return 0;
-  }
  return 0;
 }

@ -1252,11 +1217,9 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
        if (setBlockIntoRes(pInfo, &ret.data) < 0) {
          ASSERT(0);
        }
-        /*pTaskInfo->streamInfo.lastStatus = ret.offset;*/
+        // TODO clean data block
        if (pInfo->pRes->info.rows > 0) {
          return pInfo->pRes;
-          /*} else {*/
-          /*tDeleteSSDataBlock(&ret.data);*/
        }
      } else if (ret.fetchType == FETCH_TYPE__META) {
        ASSERT(0);
@ -1264,17 +1227,20 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
        pTaskInfo->streamInfo.metaBlk = ret.meta;
        return NULL;
      } else if (ret.fetchType == FETCH_TYPE__NONE) {
-        if (ret.offset.version == -1) {
-          pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG;
-          pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1;
-        } else {
-          pTaskInfo->streamInfo.lastStatus = ret.offset;
-        }
+        pTaskInfo->streamInfo.lastStatus = ret.offset;
+        ASSERT(pTaskInfo->streamInfo.lastStatus.version + 1 >= pTaskInfo->streamInfo.prepareStatus.version);
        return NULL;
      } else {
        ASSERT(0);
      }
    }
+  } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
+    SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
+    return pResult && pResult->info.rows > 0 ? pResult : NULL;
+  } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_META) {
+    // TODO scan meta
+    ASSERT(0);
+    return NULL;
  }

  size_t total = taosArrayGetSize(pInfo->pBlockLists);
@ -1386,71 +1352,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
        return NULL;
      }

-      pInfo->pRes->info.rows = block.info.rows;
-      pInfo->pRes->info.uid = block.info.uid;
-      pInfo->pRes->info.type = STREAM_NORMAL;
-      pInfo->pRes->info.capacity = block.info.rows;
+      setBlockIntoRes(pInfo, &block);

-      uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t));
-      if (groupIdPre) {
-        pInfo->pRes->info.groupId = *groupIdPre;
-      } else {
-        pInfo->pRes->info.groupId = 0;
-      }
-
-      // for generating rollup SMA result, each time is an independent time serie.
-      // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
-      if (pInfo->assignBlockUid) {
-        pInfo->pRes->info.groupId = block.info.uid;
-      }
-
-      // todo extract method
-      for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
-        SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
-        if (!pColMatchInfo->output) {
-          continue;
-        }
-
-        bool colExists = false;
-        for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) {
-          SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j);
-          if (pResCol->info.colId == pColMatchInfo->colId) {
-            taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol);
-            colExists = true;
-            break;
-          }
-        }
-
-        // the required column does not exists in submit block, let's set it to be all null value
-        if (!colExists) {
-          SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
-          colDataAppendNNULL(pDst, 0, pBlockInfo->rows);
-        }
-      }
-
-      taosArrayDestroy(block.pDataBlock);
-
-      ASSERT(pInfo->pRes->pDataBlock != NULL);
-#if 0
-      if (pInfo->pRes->pDataBlock == NULL) {
-        // TODO add log
-        updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
-        pOperator->status = OP_EXEC_DONE;
-        pTaskInfo->code = terrno;
-        return NULL;
-      }
-#endif
-
-      // currently only the tbname pseudo column
-      if (pInfo->numOfPseudoExpr > 0) {
-        code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo));
-        if (code != TSDB_CODE_SUCCESS) {
-          longjmp(pTaskInfo->env, code);
-        }
-      }
-
-      doFilter(pInfo->pCondition, pInfo->pRes);
-      blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
      if (pBlockInfo->rows > 0) {
        break;
      }
@ -1480,11 +1383,14 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {

    return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;

+#if 0
  } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) {
+    ASSERT(0);
    // check reader last status
    // if not match, reset status
    SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
    return pResult && pResult->info.rows > 0 ? pResult : NULL;
+#endif

  } else {
    ASSERT(0);
@ -1492,6 +1398,11 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
  }
 }

+static SSDataBlock* doRawScan(SOperatorInfo* pInfo) {
+  //
+  return NULL;
+}
+
 static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
  SArray* tableIdList = taosArrayInit(4, sizeof(uint64_t));

@ -1504,6 +1415,19 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
  return tableIdList;
 }

+// for subscribing db or stb (not including column),
+// if this scan is used, meta data can be return
+// and schemas are decided when scanning
+SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
+                                         SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) {
+  // create operator
+  // create tb reader
+  // create meta reader
+  // create tq reader
+
+  return NULL;
+}
+
 SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
                                            uint64_t taskId) {
@ -1547,13 +1471,16 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys

  if (pHandle) {
    SOperatorInfo*  pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
-    STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanOp->info;
+    STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info;
+    if (pHandle->version > 0) {
+      pTSInfo->cond.endVersion = pHandle->version;
+    }

    SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0);
    if (pHandle->initTableReader) {
-      pSTInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
-      pSTInfo->dataReader = NULL;
-      if (tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, &pSTInfo->dataReader, NULL) < 0) {
+      pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
+      pTSInfo->dataReader = NULL;
+      if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, tableList, &pTSInfo->dataReader, NULL) < 0) {
        ASSERT(0);
      }
    }
@ -1567,14 +1494,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
      pInfo->tqReader = pHandle->tqReader;
    }

-    if (pSTInfo->interval.interval > 0) {
-      pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark);
+    if (pTSInfo->interval.interval > 0) {
+      pInfo->pUpdateInfo = updateInfoInitP(&pTSInfo->interval, pTwSup->waterMark);
    } else {
      pInfo->pUpdateInfo = NULL;
    }

    pInfo->pTableScanOp = pTableScanOp;
-    pInfo->interval = pSTInfo->interval;
+    pInfo->interval = pTSInfo->interval;

    pInfo->readHandle = *pHandle;
    pInfo->tableUid = pScanPhyNode->uid;
@ -1873,9 +1800,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
          metaReaderInit(&mr, pInfo->readHandle.meta, 0);

          uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
-          int32_t code = metaGetTableEntryByUid(&mr, suid);
+          int32_t  code = metaGetTableEntryByUid(&mr, suid);
          if (code != TSDB_CODE_SUCCESS) {
-            qError("failed to get super table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
+            qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno),
+                   GET_TASKID(pTaskInfo));
            metaReaderClear(&mr);
            metaCloseTbCursor(pInfo->pCur);
            pInfo->pCur = NULL;
@ -2275,9 +2203,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {

  while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) {
    STableKeyInfo* item = taosArrayGet(pInfo->pTableList->pTableList, pInfo->curPos);
-    int32_t code = metaGetTableEntryByUid(&mr, item->uid);
+    int32_t        code = metaGetTableEntryByUid(&mr, item->uid);
    if (code != TSDB_CODE_SUCCESS) {
-      qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo));
+      qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno),
+             GET_TASKID(pTaskInfo));
      metaReaderClear(&mr);
      longjmp(pTaskInfo->env, terrno);
    }
@ -2565,8 +2494,8 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc

  // currently only the tbname pseudo column
  if (pTableScanInfo->numOfPseudoExpr > 0) {
-    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, pTableScanInfo->numOfPseudoExpr,
-                           pBlock, GET_TASKID(pTaskInfo));
+    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr,
+                                          pTableScanInfo->numOfPseudoExpr, pBlock, GET_TASKID(pTaskInfo));
    if (code != TSDB_CODE_SUCCESS) {
      longjmp(pTaskInfo->env, code);
    }
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@ -217,6 +217,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
      doSetOperatorCompleted(pOperator);
      break;
    }
+
    if (blockDataGetNumOfRows(pBlock) > 0) {
      break;
    }
@ -601,8 +602,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
      break;
    }

-    if (pInfo->groupSort)
-    {
+    if (pInfo->groupSort) {
      uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
      if (!pInfo->hasGroupId) {
        pInfo->groupId = tupleGroupId;
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@ -50,8 +50,11 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {

 static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) {
  // the first are always the timestamp column, so start from the second column.
-  for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
+  for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
    SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i);
+    if (p->info.type == TSDB_DATA_TYPE_TIMESTAMP && i == 0) {
+      continue;
+    }
    colDataAppendNULL(p, rowIndex);
  }
 }
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@ -59,136 +59,6 @@ static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOper

 static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; }

-static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w,
-                                      bool ascQuery) {
-  if (ascQuery) {
-    getAlignQueryTimeWindow(pInterval, precision, ts, w);
-  } else {
-    // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
-    getAlignQueryTimeWindow(pInterval, precision, ts, w);
-
-    int64_t key = w->skey;
-    while (key < ts) {  // moving towards end
-      key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision);
-      if (key >= ts) {
-        break;
-      }
-
-      w->skey = key;
-    }
-  }
-}
-
-static STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) {
-  int32_t factor = (order == TSDB_ORDER_ASC)? -1:1;
-
-  STimeWindow win = *pWindow;
-  STimeWindow save = win;
-  while(win.skey <= ts && win.ekey >= ts) {
-    save = win;
-    win.skey = taosTimeAdd(win.skey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
-    win.ekey = taosTimeAdd(win.ekey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
-  }
-
-  return save;
-}
-
-// todo do refactor
-// get the correct time window according to the handled timestamp
-STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
-                                int32_t precision, int32_t order) {
-  STimeWindow w = {0};
-
-  if (pResultRowInfo->cur.pageId == -1) {  // the first window, from the previous stored value
-    getInitialStartTimeWindow(pInterval, precision, ts, &w, true);
-    w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
-    return w;
-  }
-
-  w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win;
-
-  if (pInterval->interval == pInterval->sliding) {
-    if (w.skey > ts || w.ekey < ts) {
-      if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
-        w.skey = taosTimeTruncate(ts, pInterval, precision);
-        w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
-      } else {
-        int64_t st = w.skey;
-
-        if (st > ts) {
-          st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
-        }
-
-        int64_t et = st + pInterval->interval - 1;
-        if (et < ts) {
-          st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
-        }
-
-        w.skey = st;
-        w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
-      }
-    }
-  } else {  // it is an sliding window query, in which sliding value is not equalled to
-    // interval value, and we need to find the first qualified time window for asc/desc traverse respectively.
-    if (order == TSDB_ORDER_ASC) {
-      if (w.skey <= ts && w.ekey >= ts) {
-        // ts is resident in current time window, but we need to find the first
-        //qualified time window that cover this timestamp
-        w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order);
-      } else {
-        // todo refactor:
-        if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
-          w.skey = taosTimeTruncate(ts, pInterval, precision);
-          w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
-        } else {
-          int64_t st = w.skey;
-
-          if (st > ts) {
-            st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
-          }
-
-          int64_t et = st + pInterval->interval - 1;
-          if (et < ts) {
-            st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
-          }
-
-          w.skey = st;
-          w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
-
-          w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order);
-        }
-      }
-    } else {
-      if (w.skey <= ts && w.ekey >= ts) {
-        w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order);
-      } else {
-        // todo refactor:
-        if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
-          w.skey = taosTimeTruncate(ts, pInterval, precision);
-          w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
-        } else {
-          int64_t st = w.skey;
-
-          if (st > ts) {
-            st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
-          }
-
-          int64_t et = st + pInterval->interval - 1;
-          if (et < ts) {
-            st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
-          }
-
-          w.skey = st;
-          w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
-
-          w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order);
-        }
-      }
-    }
-  }
-  return w;
-}
-
 static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan,
                                      SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
                                      int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
@ -889,7 +759,7 @@ static void removeResults(SArray* pWins, SArray* pUpdated) {
 }

 int64_t getWinReskey(void* data, int32_t index) {
-  SArray*     res = (SArray*)data;
+  SArray*  res = (SArray*)data;
  SWinRes* pos = taosArrayGet(res, index);
  return pos->ts;
 }
@ -899,15 +769,14 @@ static void removeDeleteResults(SArray* pUpdated, SArray* pDelWins) {
  int32_t delSize = taosArrayGetSize(pDelWins);
  for (int32_t i = 0; i < upSize; i++) {
    SResKeyPos* pResKey = taosArrayGetP(pUpdated, i);
-    int64_t key = *(int64_t*)pResKey->key;
-    int32_t index = binarySearch(pDelWins, delSize, key, TSDB_ORDER_DESC, getWinReskey);
+    int64_t     key = *(int64_t*)pResKey->key;
+    int32_t     index = binarySearch(pDelWins, delSize, key, TSDB_ORDER_DESC, getWinReskey);
    if (index >= 0 && key == getWinReskey(pDelWins, index)) {
      taosArrayRemove(pDelWins, index);
    }
  }
 }

-
 bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
  ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
  return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
@ -930,8 +799,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

-  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
-                                        pInfo->interval.precision, pInfo->order);
+  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
  int32_t     ret = TSDB_CODE_SUCCESS;
  if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
@ -1206,7 +1074,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
  SOptrBasicInfo* pBInfo = &pInfo->binfo;

  if (pOperator->status == OP_RES_TO_RETURN) {
-    while(1) {
+    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      doFilter(pInfo->pCondition, pBInfo->pRes);

@ -1221,7 +1089,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
-    return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
+    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
  }

  int32_t order = TSDB_ORDER_ASC;
@ -1247,7 +1115,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {

  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
-  while(1) {
+  while (1) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
    doFilter(pInfo->pCondition, pBInfo->pRes);

@ -1262,7 +1130,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
-  return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
+  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
 }

 static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
@ -1287,11 +1155,13 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      doFilter(pInfo->pCondition, pBlock);
+
      bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
      if (!hasRemain) {
        doSetOperatorCompleted(pOperator);
        break;
      }
+
      if (pBlock->info.rows > 0) {
        break;
      }
@ -1384,13 +1254,13 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId)

 void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) {
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  TSKEY* tsStarts = (TSKEY*)pStartCol->pData;
+  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
-  uint64_t* groupIds = (uint64_t*)pGroupCol->pData;
+  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
-    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, pInterval->precision, TSDB_ORDER_ASC);
+    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
    doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
    if (pUpWins) {
      SWinRes winRes = {.ts = win.skey, .groupId = groupIds[i]};
@ -1399,8 +1269,8 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock,
  }
 }

-static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval,
-                           int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) {
+static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
+                           SSDataBlock* pBlock, SArray* pUpWins) {
  SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
  TSKEY*           tsCols = (TSKEY*)pTsCol->pData;
  uint64_t*        pGpDatas = NULL;
@ -1412,10 +1282,10 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval*
  for (int32_t i = 0; i < pBlock->info.rows; i += step) {
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
-    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, TSDB_ORDER_ASC);
+    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, TSDB_ORDER_ASC);
    step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
    uint64_t winGpId = pGpDatas ? pGpDatas[i] : pBlock->info.groupId;
-    bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), winGpId, numOfOutput);
+    bool     res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), winGpId, numOfOutput);
    if (pUpWins && res) {
      SWinRes winRes = {.ts = win.skey, .groupId = winGpId};
      taosArrayPush(pUpWins, &winRes);
@ -1440,9 +1310,9 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
  return TSDB_CODE_SUCCESS;
 }

-static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
-    SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins,
-    SArray* pRecyPages, SDiskbasedBuf* pDiscBuf) {
+static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
+                                   SHashObj* pPullDataMap, SArray* closeWins, SArray* pRecyPages,
+                                   SDiskbasedBuf* pDiscBuf) {
  void*  pIte = NULL;
  size_t keyLen = 0;
  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
@ -1452,7 +1322,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
    SResultRowInfo dumyInfo;
    dumyInfo.cur.pageId = -1;
-    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, TSDB_ORDER_ASC);
+    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, TSDB_ORDER_ASC);
    SWinRes     winRe = {
            .ts = win.skey,
            .groupId = groupId,
@ -1497,8 +1367,8 @@ static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) {
    SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
    ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);
    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
-    closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup,
-        &pChInfo->interval, NULL, NULL, NULL, pChInfo->aggSup.pResultBuf);
+    closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL,
+                        pChInfo->aggSup.pResultBuf);
  }
 }

@ -1544,7 +1414,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
  }

  if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex ,pInfo->pDelRes);
+    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
    if (pInfo->pDelRes->info.rows > 0) {
      return pInfo->pDelRes;
    }
@ -1559,7 +1429,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {

  SOperatorInfo* downstream = pOperator->pDownstream[0];

-  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos
+  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
  while (1) {
    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
    if (pBlock == NULL) {
@ -1568,11 +1438,12 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
    printDataBlock(pBlock, "single interval recv");

    if (pBlock->info.type == STREAM_CLEAR) {
-      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval,
-          pOperator->exprSupp.numOfExprs, pBlock, NULL);
+      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
+                     NULL);
      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
      continue;
-    } if (pBlock->info.type == STREAM_DELETE_DATA) { 
+    }
+    if (pBlock->info.type == STREAM_DELETE_DATA) {
      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
      continue;
    } else if (pBlock->info.type == STREAM_GET_ALL) {
@ -1597,8 +1468,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
  }
  pOperator->status = OP_RES_TO_RETURN;
-  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
-       &pInfo->interval, NULL, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
+  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated,
+                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);

  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
@ -1793,8 +1664,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
  pInfo->delIndex = 0;
  // pInfo->pDelRes = createPullDataBlock(); todo(liuyao) for delete
-  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
-  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
+  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

@ -1962,7 +1833,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
  SExprSupp*               pSup = &pOperator->exprSupp;

  if (pOperator->status == OP_RES_TO_RETURN) {
-    while(1) {
+    while (1) {
      doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
      doFilter(pInfo->pCondition, pBInfo->pRes);

@ -1977,7 +1848,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
      }
    }
    pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
-    return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
+    return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
  }

  int64_t st = taosGetTimestampUs();
@ -2006,7 +1877,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {

  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
-  while(1) {
+  while (1) {
    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
    doFilter(pInfo->pCondition, pBInfo->pRes);

@ -2021,7 +1892,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
    }
  }
  pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
-  return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
+  return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
 }

 static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
@ -2391,7 +2262,8 @@ void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {

 SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                            SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
-                                            STimeWindowAggSupp* pTwAggSupp, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
+                                            STimeWindowAggSupp* pTwAggSupp, SNode* pCondition,
+                                            SExecTaskInfo* pTaskInfo) {
  SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (pInfo == NULL || pOperator == NULL) {
@ -2470,20 +2342,20 @@ bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
  return p1 != NULL;
 }

-static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup,
-    SArray* pWinArray, int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) {
+static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
+                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) {
  int32_t size = taosArrayGetSize(pWinArray);
  if (!pInfo->pChildren) {
    return;
  }
  for (int32_t i = 0; i < size; i++) {
-    SWinRes* pWinRes = taosArrayGet(pWinArray, i);
-    SResultRow*  pCurResult = NULL;
-    STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts+1};
-    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput,
-                           pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
+    SWinRes*    pWinRes = taosArrayGet(pWinArray, i);
+    SResultRow* pCurResult = NULL;
+    STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts + 1};
+    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
+                           numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
    int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
-    bool find = true;
+    bool    find = true;
    for (int32_t j = 0; j < numOfChildren; j++) {
      SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
      SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
@ -2493,8 +2365,9 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr
      }
      find = true;
      SResultRow* pChResult = NULL;
-      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId, pChildSup->pCtx,
-                             pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, pTaskInfo);
+      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId,
+                             pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
+                             pTaskInfo);
      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo);
    }
    if (find && pUpdated) {
@ -2528,6 +2401,12 @@ void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) {

 static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

+STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
+  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
+  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
+  return w;
+}
+
 static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
                           SArray* pUpdated) {
  SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
@ -2547,8 +2426,12 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc

  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
-  STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
-                                            pInfo->interval.precision, pInfo->order);
+  STimeWindow nextWin = {0};
+  if (IS_FINAL_OP(pInfo)) {
+    nextWin = getFinalTimeWindow(ts, &pInfo->interval);
+  } else {
+    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order);
+  }
  while (1) {
    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
    if (pInfo->ignoreExpiredData && isClosed) {
@ -2605,8 +2488,12 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
      longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
    }

-    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
+    if (IS_FINAL_OP(pInfo)) {
+      forwardRows = 1;
+    } else {
+      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                           TSDB_ORDER_ASC);
+    }
    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
      saveResultRow(pResult, tableGroupId, pUpdated);
    }
@ -2772,7 +2659,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
      clearSpecialDataBlock(pInfo->pUpdateRes);
      removeDeleteResults(pUpdated, pInfo->pDelWins);
      pOperator->status = OP_RES_TO_RETURN;
-      qInfo("Stream Final Interval return data");
+      qInfo("%s return data", IS_FINAL_OP(pInfo) ? "interval Final" : "interval  Semi");
      break;
    }
    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval  Semi recv");
@ -2783,18 +2670,16 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
      pInfo->binfo.pRes->info.type = pBlock->info.type;
    } else if (pBlock->info.type == STREAM_CLEAR) {
      SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes));
-      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs,
-                     pBlock, pUpWins);
+      doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins);
      if (IS_FINAL_OP(pInfo)) {
        int32_t                           childIndex = getChildIndex(pBlock);
        SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
        SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;

-        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval,
-                       pChildSup->numOfExprs, pBlock, NULL);
-        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId,
-            pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, NULL);
+        doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL);
+        rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs,
+                              pOperator->pTaskInfo, NULL);
        taosArrayDestroy(pUpWins);
        continue;
      }
@ -2812,7 +2697,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
        SExprSupp*                        pChildSup = &pChildOp->exprSupp;
        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval);
        rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId,
-            pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated);
+                              pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated);
        continue;
      }
      removeResults(pInfo->pDelWins, pUpdated);
@ -2834,6 +2719,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
      continue;
    }

+    if (pInfo->scalarSupp.pExprInfo != NULL) {
+      SExprSupp* pExprSup = &pInfo->scalarSupp;
+      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    }
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated);
    if (IS_FINAL_OP(pInfo)) {
@ -2862,8 +2751,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
  if (IS_FINAL_OP(pInfo)) {
-    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
-        &pInfo->interval, pInfo->pPullDataMap, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
+    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap,
+                        pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
    closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);
  }

@ -2951,6 +2840,15 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
  initResultSizeInfo(pOperator, 4096);
+  if (pIntervalPhyNode->window.pExprs != NULL) {
+    int32_t    numOfScalar = 0;
+    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
+    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+  }
+
  int32_t      numOfCols = 0;
  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
@ -3005,8 +2903,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
  pInfo->pPullDataRes = createPullDataBlock();
  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
  // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete
-  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
-  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
+  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
  pInfo->delIndex = 0;
  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));

@ -3117,6 +3015,14 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
  }

  initResultSizeInfo(pOperator, 4096);
+  if (pSessionNode->window.pExprs != NULL) {
+    int32_t    numOfScalar = 0;
+    SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
+    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+  }
  SExprSupp* pSup = &pOperator->exprSupp;

  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
@ -3152,8 +3058,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
  pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  // pInfo->pDelRes = createPullDataBlock();
-  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
-  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
+  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
  pInfo->pChildren = NULL;
  pInfo->isFinal = false;
  pInfo->pPhyNode = pPhyNode;
@ -3206,9 +3112,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
  return false;
 }

-bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) {
-  return isInTimeWindow(&pWinInfo->win, ts, gap);
-}
+bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }

 static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
@ -3234,7 +3138,7 @@ SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) {

 // don't add new window
 SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
-                                        int64_t gap, int32_t* pIndex) {
+                                       int64_t gap, int32_t* pIndex) {
  SArray* pWinInfos = getWinInfos(pAggSup, groupId);
  pAggSup->pCurWins = pWinInfos;

@ -3243,7 +3147,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start
    return NULL;
  }
  // find the first position which is smaller than the key
-  int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
+  int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
  SResultWindowInfo* pWin = NULL;
  if (index >= 0) {
    pWin = taosArrayGet(pWinInfos, index);
@ -3514,16 +3418,15 @@ void deleteWindow(SArray* pWinInfos, int32_t index) {

 static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) {
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  TSKEY* startDatas = (TSKEY*)pStartTsCol->pData;
+  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
-  TSKEY* endDatas = (TSKEY*)pEndTsCol->pData;
+  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
-  uint64_t* gpDatas = (uint64_t*)pGroupCol->pData;
+  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    int32_t winIndex = 0;
-    while(1) {
-      SResultWindowInfo* pCurWin =
-          getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
+    while (1) {
+      SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex);
      if (!pCurWin) {
        break;
      }
@ -3573,6 +3476,7 @@ static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) {
    *(int64_t*)pos->key = ((SWinRes*)pData)->ts;
    taosArrayPush(pUpdated, &pos);
  }
+  taosArraySort(pUpdated, resultrowComparAsc);
  return TSDB_CODE_SUCCESS;
 }

@ -3755,8 +3659,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {

    if (pBlock->info.type == STREAM_CLEAR) {
      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
-      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs,
-                            0, pWins);
+      doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, 0,
+                            pWins);
      if (IS_FINAL_OP(pInfo)) {
        int32_t                        childIndex = getChildIndex(pBlock);
        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
@ -3787,6 +3691,10 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
      continue;
    }

+    if (pInfo->scalarSupp.pExprInfo != NULL) {
+      SExprSupp* pExprSup = &pInfo->scalarSupp;
+      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    }
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo));
@ -3919,6 +3827,10 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
      continue;
    }

+    if (pInfo->scalarSupp.pExprInfo != NULL) {
+      SExprSupp* pExprSup = &pInfo->scalarSupp;
+      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    }
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false);
@ -4315,6 +4227,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
      continue;
    }

+    if (pInfo->scalarSupp.pExprInfo != NULL) {
+      SExprSupp* pExprSup = &pInfo->scalarSupp;
+      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    }
    // the pDataBlock are always the same one, no need to call this again
    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
    doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
@ -4368,6 +4284,15 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);
  initResultSizeInfo(pOperator, 4096);
+  if (pStateNode->window.pExprs != NULL) {
+    int32_t    numOfScalar = 0;
+    SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
+    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+  }
+
  initResultRowInfo(&pInfo->binfo.resultRowInfo);
  pInfo->twAggSup = (STimeWindowAggSupp){
      .waterMark = pStateNode->window.watermark,
@ -4398,8 +4323,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
  pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
  pInfo->pDelIterator = NULL;
  // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete
-  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete
-  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete
+  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
+  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
  pInfo->pChildren = NULL;
  pInfo->ignoreExpiredData = pStateNode->window.igExpired;

@ -4428,17 +4353,6 @@ _error:
  return NULL;
 }

-typedef struct SMergeAlignedIntervalAggOperatorInfo {
-  SIntervalAggOperatorInfo *intervalAggOperatorInfo;
-
-  bool         hasGroupId;
-  uint64_t     groupId;
-  SSDataBlock* prefetchedBlock;
-  bool         inputBlocksFinished;
-
-  SNode*       pCondition;
-} SMergeAlignedIntervalAggOperatorInfo;
-
 void destroyMergeAlignedIntervalOperatorInfo(void* param, int32_t numOfOutput) {
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param;
  destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo, numOfOutput);
@ -4514,7 +4428,8 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
      currTs = tsCols[currPos];
      currWin.skey = currTs;
      currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit,
-                                 iaInfo->interval.precision) - 1;
+                                 iaInfo->interval.precision) -
+                     1;
      startPos = currPos;
      ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                   pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo);
@ -4587,13 +4502,15 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
  }

  size_t rows = pRes->info.rows;
+  blockDataUpdateTsWindow(pRes, iaInfo->primaryTsIndex);
  pOperator->resultInfo.totalRows += rows;
  return (rows == 0) ? NULL : pRes;
 }

 SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo,
                                                      int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval,
-                                                      int32_t primaryTsSlotId, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
+                                                      int32_t primaryTsSlotId, SNode* pCondition,
+                                                      SExecTaskInfo* pTaskInfo) {
  SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo));
  SOperatorInfo*                        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
  if (miaInfo == NULL || pOperator == NULL) {
@ -4725,7 +4642,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t
      continue;
    }
    STimeWindow* prevWin = &prevGrpWin->window;
-    if ((ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) {
+    if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) {
      finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock);
      tdListPopNode(miaInfo->groupIntervals, listNode);
    }
@ -4750,8 +4667,8 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo*
  TSKEY       blockStartTs = getStartTsKey(&pBlock->info.window, tsCols);
  SResultRow* pResult = NULL;

-  STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval,
-                                        iaInfo->interval.precision, iaInfo->order);
+  STimeWindow win =
+      getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, iaInfo->order);

  int32_t ret =
      setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx,
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@ -48,8 +48,8 @@ static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
    return TIME_UNIT_INVALID;
  }

-  if (TSDB_TIME_PRECISION_MILLI == dbPrec && (0 == strcasecmp(pVal->literal, "1u") ||
-                                              0 == strcasecmp(pVal->literal, "1b"))) {
+  if (TSDB_TIME_PRECISION_MILLI == dbPrec &&
+      (0 == strcasecmp(pVal->literal, "1u") || 0 == strcasecmp(pVal->literal, "1b"))) {
    return TIME_UNIT_TOO_SMALL;
  }

@ -57,10 +57,9 @@ static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
    return TIME_UNIT_TOO_SMALL;
  }

-  if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' &&
-                                  pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
-                                  pVal->literal[1] != 'h' && pVal->literal[1] != 'd' &&
-                                  pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) {
+  if (pVal->literal[0] != '1' ||
+      (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' && pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
+       pVal->literal[1] != 'h' && pVal->literal[1] != 'd' && pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) {
    return TIME_UNIT_INVALID;
  }

@ -678,9 +677,10 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
  }

-  uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
-  if (TSDB_DATA_TYPE_TIMESTAMP != paraType) {
-    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  SNode* pPara1 = nodesListGetNode(pFunc->pParameterList, 0);
+  if (QUERY_NODE_COLUMN != nodeType(pPara1) || PRIMARYKEY_TIMESTAMP_COL_ID != ((SColumnNode*)pPara1)->colId) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "The first parameter of the ELAPSED function can only be the timestamp primary key");
  }

  // param1
@ -694,8 +694,7 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len

    pValue->notReserved = true;

-    paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
-    if (!IS_INTEGER_TYPE(paraType)) {
+    if (!IS_INTEGER_TYPE(pValue->node.resType.type)) {
      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
    }

@ -706,8 +705,9 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                             "ELAPSED function time unit parameter should be greater than db precision");
    } else if (ret == TIME_UNIT_INVALID) {
-      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                             "ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+      return buildFuncErrMsg(
+          pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+          "ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
    }
  }

@ -1229,7 +1229,8 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
                             "STATEDURATION function time unit parameter should be greater than db precision");
    } else if (ret == TIME_UNIT_INVALID) {
      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                             "STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+                             "STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, "
+                             "1s, 1m, 1h, 1d, 1w]");
    }
  }

@ -1740,8 +1741,9 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                           "TIMETRUNCATE function time unit parameter should be greater than db precision");
  } else if (ret == TIME_UNIT_INVALID) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+    return buildFuncErrMsg(
+        pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+        "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
  }

  addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
@ -1779,8 +1781,9 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
                             "TIMEDIFF function time unit parameter should be greater than db precision");
    } else if (ret == TIME_UNIT_INVALID) {
-      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                             "TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
+      return buildFuncErrMsg(
+          pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+          "TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
    }
  }

--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@ -1863,8 +1863,6 @@ static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) {
  }

  pOutput->count += pInput->count;
-
-  return;
 }

 int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) {
@ -1874,14 +1872,13 @@ int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) {

  SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));

-  int32_t     start = pInput->startRowIndex;
-  char*       data = colDataGetData(pCol, start);
-  SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data);
-
-  stddevTransferInfo(pInputInfo, pInfo);
+  for(int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
+    char*       data = colDataGetData(pCol, i);
+    SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data);
+    stddevTransferInfo(pInputInfo, pInfo);
+  }

  SET_VAL(GET_RES_INFO(pCtx), 1, 1);
-
  return TSDB_CODE_SUCCESS;
 }

--- a/source/libs/function/src/functionMgt.c
+++ b/source/libs/function/src/functionMgt.c
@ -139,7 +139,7 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG

 bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); }

-bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId); }
+bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId) && !fmIsPseudoColumnFunc(funcId); }

 bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); }

--- a/source/libs/index/inc/indexFstCommon.h
+++ b/source/libs/index/inc/indexFstCommon.h
@ -4,6 +4,7 @@
 #include "tutil.h"
 extern const uint8_t COMMON_INPUTS[];
 extern const char    COMMON_INPUTS_INV[];
+extern const int32_t COMMON_INPUTS_LEN;

 #ifdef __cplusplus
 extern "C" {
--- a/source/libs/index/inc/indexFstDfa.h
+++ b/source/libs/index/inc/indexFstDfa.h
@ -29,16 +29,16 @@ extern "C" {
 typedef struct FstDfa FstDfa;

 typedef struct {
-  SArray * insts;
+  SArray  *insts;
  uint32_t next[256];
  bool     isMatch;
-} State;
+} DfaState;

 /*
 * dfa builder related func
 **/
 typedef struct FstDfaBuilder {
-  FstDfa *  dfa;
+  FstDfa   *dfa;
  SHashObj *cache;
 } FstDfaBuilder;

@ -51,7 +51,7 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder);
 bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t bytes,
                        uint32_t *result);

-bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result);
+bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result);

 /*
 * dfa related func
--- a/source/libs/index/inc/indexFstRegex.h
+++ b/source/libs/index/inc/indexFstRegex.h
@ -65,6 +65,7 @@ typedef struct {
 } FstRegex;

 FstRegex *regexCreate(const char *str);
+void      regexDestroy(FstRegex *regex);

 uint32_t regexAutomStart(FstRegex *regex);
 bool     regexAutomIsMatch(FstRegex *regex, uint32_t state);
--- a/source/libs/index/inc/indexFstSparse.h
+++ b/source/libs/index/inc/indexFstSparse.h
@ -23,17 +23,18 @@ extern "C" {
 #endif

 typedef struct FstSparseSet {
-  uint32_t *dense;
-  uint32_t *sparse;
-  int32_t   size;
+  int32_t *dense;
+  int32_t *sparse;
+  int32_t  size;
+  int32_t  cap;
 } FstSparseSet;

 FstSparseSet *sparSetCreate(int32_t sz);
 void          sparSetDestroy(FstSparseSet *s);
 uint32_t      sparSetLen(FstSparseSet *ss);
-uint32_t      sparSetAdd(FstSparseSet *ss, uint32_t ip);
-uint32_t      sparSetGet(FstSparseSet *ss, uint32_t i);
-bool          sparSetContains(FstSparseSet *ss, uint32_t ip);
+bool          sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *val);
+bool          sparSetGet(FstSparseSet *ss, int32_t i, int32_t *val);
+bool          sparSetContains(FstSparseSet *ss, int32_t ip);
 void          sparSetClear(FstSparseSet *ss);

 #ifdef __cplusplus
--- a/source/libs/index/src/indexCache.c
+++ b/source/libs/index/src/indexCache.c
@ -22,7 +22,7 @@
 #define MAX_INDEX_KEY_LEN 256  // test only, change later

 #define MEM_TERM_LIMIT     10 * 10000
-#define MEM_THRESHOLD      64 * 1024
+#define MEM_THRESHOLD      512 * 1024
 #define MEM_SIGNAL_QUIT    MEM_THRESHOLD * 20
 #define MEM_ESTIMATE_RADIO 1.5

@ -204,7 +204,6 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
    if (0 == strcmp(c->colVal, pCt->colVal)) {
      if (c->operaType == ADD_VALUE) {
        INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
-        // taosArrayPush(result, &c->uid);
        *s = kTypeValue;
      } else if (c->operaType == DEL_VALUE) {
        INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
@ -309,7 +308,6 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
    if (cond == MATCH) {
      if (c->operaType == ADD_VALUE) {
        INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
-        // taosArrayPush(result, &c->uid);
        *s = kTypeValue;
      } else if (c->operaType == DEL_VALUE) {
        INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
--- a/source/libs/index/src/indexFst.c
+++ b/source/libs/index/src/indexFst.c
@ -1307,7 +1307,6 @@ FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) {
    taosArrayPush(sws->inp, &(trn.inp));

    if (FST_NODE_IS_FINAL(nextNode)) {
-      // void *eofState = sws->aut->acceptEof(nextState);
      void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
      if (eofState != NULL) {
        isMatch = automFuncs[aut->type].isMatch(aut, eofState);
--- a/source/libs/index/src/indexFstCommon.c
+++ b/source/libs/index/src/indexFstCommon.c
@ -294,3 +294,4 @@ const char COMMON_INPUTS_INV[] = {
    '\xee', '\xef', '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb',
    '\xfc', '\xfd', '\xfe', '\xff',
 };
+const int32_t COMMON_INPUTS_LEN = sizeof(COMMON_INPUTS) / sizeof(COMMON_INPUTS[0]);
--- a/source/libs/index/src/indexFstDfa.c
+++ b/source/libs/index/src/indexFstDfa.c
@ -41,7 +41,7 @@ FstDfaBuilder *dfaBuilderCreate(SArray *insts) {
    return NULL;
  }

-  SArray *states = taosArrayInit(4, sizeof(State));
+  SArray *states = taosArrayInit(4, sizeof(DfaState));

  builder->dfa = dfaCreate(insts, states);
  builder->cache = taosHashInit(
@ -64,16 +64,16 @@ void dfaBuilderDestroy(FstDfaBuilder *builder) {
  taosMemoryFree(builder);
 }

-FstDfa *dfaBuilder(FstDfaBuilder *builder) {
+FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) {
  uint32_t      sz = taosArrayGetSize(builder->dfa->insts);
  FstSparseSet *cur = sparSetCreate(sz);
  FstSparseSet *nxt = sparSetCreate(sz);

  dfaAdd(builder->dfa, cur, 0);

-  SArray * states = taosArrayInit(0, sizeof(uint32_t));
+  SArray  *states = taosArrayInit(0, sizeof(uint32_t));
  uint32_t result;
-  if (dfaBuilderCachedState(builder, cur, &result)) {
+  if (dfaBuilderCacheState(builder, cur, &result)) {
    taosArrayPush(states, &result);
  }
  SHashObj *seen = taosHashInit(12, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
@ -101,17 +101,18 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) {
 bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte,
                        uint32_t *result) {
  sparSetClear(cur);
-  State *t = taosArrayGet(builder->dfa->states, state);
+  DfaState *t = taosArrayGet(builder->dfa->states, state);
  for (int i = 0; i < taosArrayGetSize(t->insts); i++) {
-    uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
-    sparSetAdd(cur, ip);
+    int32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
+    bool    succ = sparSetAdd(cur, ip, NULL);
+    assert(succ == true);
  }
  dfaRun(builder->dfa, cur, next, byte);

  t = taosArrayGet(builder->dfa->states, state);

  uint32_t nxtState;
-  if (dfaBuilderCachedState(builder, next, &nxtState)) {
+  if (dfaBuilderCacheState(builder, next, &nxtState)) {
    t->next[byte] = nxtState;
    *result = nxtState;
    return true;
@ -119,12 +120,13 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet
  return false;
 }

-bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) {
+bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) {
  SArray *tinsts = taosArrayInit(4, sizeof(uint32_t));
  bool    isMatch = false;

  for (int i = 0; i < sparSetLen(set); i++) {
-    uint32_t ip = sparSetGet(set, i);
+    int32_t ip;
+    if (false == sparSetGet(set, i, &ip)) continue;

    Inst *inst = taosArrayGet(builder->dfa->insts, ip);
    if (inst->ty == JUMP || inst->ty == SPLIT) {
@ -144,7 +146,7 @@ bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *
    *result = *v;
    taosArrayDestroy(tinsts);
  } else {
-    State st;
+    DfaState st;
    st.insts = tinsts;
    st.isMatch = isMatch;
    taosArrayPush(builder->dfa->states, &st);
@ -169,14 +171,14 @@ bool dfaIsMatch(FstDfa *dfa, uint32_t si) {
  if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
    return false;
  }
-  State *st = taosArrayGet(dfa->states, si);
+  DfaState *st = taosArrayGet(dfa->states, si);
  return st != NULL ? st->isMatch : false;
 }
 bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result) {
  if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
    return false;
  }
-  State *st = taosArrayGet(dfa->states, si);
+  DfaState *st = taosArrayGet(dfa->states, si);
  *result = st->next[byte];
  return true;
 }
@ -184,7 +186,8 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) {
  if (sparSetContains(set, ip)) {
    return;
  }
-  sparSetAdd(set, ip);
+  bool succ = sparSetAdd(set, ip, NULL);
+  // assert(succ == true);
  Inst *inst = taosArrayGet(dfa->insts, ip);
  if (inst->ty == MATCH || inst->ty == RANGE) {
    // do nothing
@ -201,7 +204,8 @@ bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) {
  bool isMatch = false;
  sparSetClear(to);
  for (int i = 0; i < sparSetLen(from); i++) {
-    uint32_t ip = sparSetGet(from, i);
+    int32_t ip;
+    if (false == sparSetGet(from, i, &ip)) continue;

    Inst *inst = taosArrayGet(dfa->insts, ip);
    if (inst->ty == JUMP || inst->ty == SPLIT) {
--- a/source/libs/index/src/indexFstRegex.c
+++ b/source/libs/index/src/indexFstRegex.c
@ -22,20 +22,26 @@ FstRegex *regexCreate(const char *str) {
  if (regex == NULL) {
    return NULL;
  }
-  int32_t sz = (int32_t)strlen(str);
-  char *  orig = taosMemoryCalloc(1, sz);
-  memcpy(orig, str, sz);

-  regex->orig = orig;
+  regex->orig = tstrdup(str);

  // construct insts based on str
-  SArray *insts = NULL;
-
+  SArray *insts = taosArrayInit(256, sizeof(uint8_t));
+  for (int i = 0; i < strlen(str); i++) {
+    uint8_t v = str[i];
+    taosArrayPush(insts, &v);
+  }
  FstDfaBuilder *builder = dfaBuilderCreate(insts);
  regex->dfa = dfaBuilderBuild(builder);
  return regex;
 }

+void regexDestroy(FstRegex *regex) {
+  if (regex == NULL) return;
+  taosMemoryFree(regex->orig);
+  taosMemoryFree(regex);
+}
+
 uint32_t regexAutomStart(FstRegex *regex) {
  ///// no nothing
  return 0;
--- a/source/libs/index/src/indexFstSparse.c
+++ b/source/libs/index/src/indexFstSparse.c
@ -15,14 +15,24 @@

 #include "indexFstSparse.h"

+static void sparSetUtil(int32_t *buf, int32_t cap) {
+  for (int32_t i = 0; i < cap; i++) {
+    buf[i] = -1;
+  }
+}
 FstSparseSet *sparSetCreate(int32_t sz) {
  FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet));
  if (ss == NULL) {
    return NULL;
  }

-  ss->dense = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
-  ss->sparse = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
+  ss->dense = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t));
+  ss->sparse = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t));
+  sparSetUtil(ss->dense, sz);
+  sparSetUtil(ss->sparse, sz);
+
+  ss->cap = sz;
+
  ss->size = 0;
  return ss;
 }
@ -38,23 +48,39 @@ uint32_t sparSetLen(FstSparseSet *ss) {
  // Get occupied size
  return ss == NULL ? 0 : ss->size;
 }
-uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) {
+bool sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *idx) {
  if (ss == NULL) {
-    return 0;
+    return false;
+  }
+  if (ip >= ss->cap || ip < 0) {
+    return false;
  }
  uint32_t i = ss->size;
  ss->dense[i] = ip;
  ss->sparse[ip] = i;
  ss->size += 1;
-  return i;
+
+  if (idx != NULL) *idx = i;
+
+  return true;
 }
-uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) {
-  // check later
-  return ss->dense[i];
+bool sparSetGet(FstSparseSet *ss, int32_t idx, int32_t *ip) {
+  if (idx >= ss->cap || idx >= ss->size || idx < 0) {
+    return false;
+  }
+  int32_t val = ss->dense[idx];
+  if (ip != NULL) {
+    *ip = val;
+  }
+  return val == -1 ? false : true;
 }
-bool sparSetContains(FstSparseSet *ss, uint32_t ip) {
-  uint32_t i = ss->sparse[ip];
-  if (i < ss->size && ss->dense[i] == ip) {
+bool sparSetContains(FstSparseSet *ss, int32_t ip) {
+  if (ip >= ss->cap || ip < 0) {
+    return false;
+  }
+  int32_t i = ss->sparse[ip];
+
+  if (i >= 0 && i < ss->cap && i < ss->size && ss->dense[i] == ip) {
    return true;
  } else {
    return false;
@ -64,5 +90,7 @@ void sparSetClear(FstSparseSet *ss) {
  if (ss == NULL) {
    return;
  }
+  sparSetUtil(ss->dense, ss->cap);
+  sparSetUtil(ss->sparse, ss->cap);
  ss->size = 0;
 }
--- a/source/libs/index/test/CMakeLists.txt
+++ b/source/libs/index/test/CMakeLists.txt
@ -4,6 +4,7 @@ IF(NOT TD_DARWIN)
  add_executable(idxFstUT "")
  add_executable(idxUtilUT "")
  add_executable(idxJsonUT "")
+  add_executable(idxFstUtilUT "")

  target_sources(idxTest
    PRIVATE 
@ -23,6 +24,25 @@ IF(NOT TD_DARWIN)
    "utilUT.cc" 
  )

+  target_sources(idxJsonUT
+    PRIVATE 
+    "jsonUT.cc" 
+  )
+  target_sources(idxFstUtilUT
+   PRIVATE 
+   "fstUtilUT.cc" 
+  )
+ 
+  target_include_directories (idxTest
+   PUBLIC
+   "${TD_SOURCE_DIR}/include/libs/index" 
+   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
+  target_include_directories (idxFstTest 
+   PUBLIC
+  "${TD_SOURCE_DIR}/include/libs/index" 
+  "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
  target_sources(idxJsonUT
    PRIVATE 
    "jsonUT.cc" 
@ -50,6 +70,38 @@ IF(NOT TD_DARWIN)
    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
  ) 

+  target_include_directories (idxJsonUT
+   PUBLIC
+   "${TD_SOURCE_DIR}/include/libs/index" 
+   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
+  target_include_directories (idxFstUtilUT
+   PUBLIC
+   "${TD_SOURCE_DIR}/include/libs/index" 
+   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+  ) 
+
+  target_link_libraries (idxTest
+    os  
+    util
+    common
+    gtest_main
+    index
+  )
+  target_link_libraries (idxFstTest
+    os  
+    util
+    common
+    gtest_main
+    index
+  )
+  target_link_libraries (idxFstUT
+    os  
+    util
+    common
+    gtest_main
+    index
+  )
  target_include_directories (idxJsonUT
    PUBLIC
    "${TD_SOURCE_DIR}/include/libs/index" 
@ -92,15 +144,28 @@ IF(NOT TD_DARWIN)
    gtest_main
    index
  )
+  target_link_libraries (idxFstUtilUT
+    os  
+    util
+    common
+    gtest_main
+    index
+  )
+  
+  add_test(
+    NAME idxJsonUT
+    COMMAND idxJsonUT 
+  )
+  add_test(
+    NAME idxFstUtilUT 
+    COMMAND idxFstUtilUT  
+  
+  )

  add_test(
    NAME idxtest
    COMMAND idxTest
  )
-  add_test(
-    NAME idxJsonUT
-    COMMAND idxJsonUT 
-  )
  add_test(
    NAME idxUtilUT 
    COMMAND idxUtilUT 
--- a/source/libs/index/test/fstUtilUT.cc
+++ b/source/libs/index/test/fstUtilUT.cc
@ -0,0 +1,112 @@
+
+#include <gtest/gtest.h>
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+#include "index.h"
+#include "indexCache.h"
+#include "indexFst.h"
+#include "indexFstDfa.h"
+#include "indexFstRegex.h"
+#include "indexFstSparse.h"
+#include "indexFstUtil.h"
+#include "indexInt.h"
+#include "indexTfile.h"
+#include "tglobal.h"
+#include "tlog.h"
+#include "tskiplist.h"
+#include "tutil.h"
+class FstUtilEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {
+    SArray *inst = taosArrayInit(4, sizeof(char));
+    builder = dfaBuilderCreate(inst);
+  }
+  virtual void TearDown() { dfaBuilderDestroy(builder); }
+
+  FstDfaBuilder *builder;
+};
+
+class FstRegexEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() { regex = regexCreate("test"); }
+  virtual void TearDown() { regexDestroy(regex); }
+  FstRegex    *regex;
+};
+
+class FstSparseSetEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() { set = sparSetCreate(256); }
+  virtual void TearDown() {
+    // tear down
+    sparSetDestroy(set);
+  }
+  void ReBuild(int32_t sz) {
+    sparSetDestroy(set);
+    set = sparSetCreate(sz);
+  }
+  FstSparseSet *set;
+};
+
+// test FstDfaBuilder
+TEST_F(FstUtilEnv, test1) {
+  // test
+}
+TEST_F(FstUtilEnv, test2) {
+  // test
+}
+TEST_F(FstUtilEnv, test3) {
+  // test
+}
+TEST_F(FstUtilEnv, test4) {
+  // test
+}
+
+// test FstRegex
+
+TEST_F(FstRegexEnv, test1) {
+  //
+  EXPECT_EQ(regex != NULL, true);
+}
+TEST_F(FstRegexEnv, test2) {}
+TEST_F(FstRegexEnv, test3) {}
+TEST_F(FstRegexEnv, test4) {}
+
+// test FstSparseSet
+TEST_F(FstSparseSetEnv, test1) {
+  for (int8_t i = 0; i < 20; i++) {
+    int32_t val = -1;
+    bool    succ = sparSetAdd(set, 'a' + i, &val);
+  }
+  EXPECT_EQ(sparSetLen(set), 20);
+  for (int8_t i = 0; i < 20; i++) {
+    int  val = -1;
+    bool find = sparSetGet(set, i, &val);
+    EXPECT_EQ(find, true);
+    EXPECT_EQ(val, i + 'a');
+  }
+  for (int8_t i = 'a'; i < 'a' + 20; i++) {
+    EXPECT_EQ(sparSetContains(set, i), true);
+  }
+
+  for (int8_t i = 'A'; i < 20; i++) {
+    EXPECT_EQ(sparSetContains(set, 'A'), false);
+  }
+
+  for (int i = 512; i < 1000; i++) {
+    EXPECT_EQ(sparSetAdd(set, i, NULL), false);
+
+    EXPECT_EQ(sparSetGet(set, i, NULL), false);
+    EXPECT_EQ(sparSetContains(set, i), false);
+  }
+  sparSetClear(set);
+
+  for (int i = 'a'; i < 'a' + 20; i++) {
+    EXPECT_EQ(sparSetGet(set, i, NULL), false);
+  }
+  for (int i = 1000; i < 2000; i++) {
+    EXPECT_EQ(sparSetGet(set, i, NULL), false);
+  }
+}
--- a/Show More
+++ b/Show More