Merge branch '3.0' into test/count_partition

2022-07-11 20:11:45 +08:00 · 2022-07-11 20:11:45 +08:00 · 877ac548f6
parent 12e56b2f97 a52c089d82
commit 877ac548f6
267 changed files with 11806 additions and 7298 deletions
--- a/docs/en/12-taos-sql/02-database.md
+++ b/docs/en/12-taos-sql/02-database.md
@ -32,7 +32,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
   - cacheLast: [Description](/reference/config/#cachelast)
   - replica: [Description](/reference/config/#replica)
   - quorum: [Description](/reference/config/#quorum)
-   - maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb)
   - comp: [Description](/reference/config/#comp)
   - precision: [Description](/reference/config/#precision)
 6. Please note that all of the parameters mentioned in this section are configured in configuration file `taos.cfg` on the TDengine server. If not specified in the `create database` statement, the values from taos.cfg are used by default. To override default parameters, they must be specified in the `create database` statement.
--- a/docs/zh/12-taos-sql/02-database.md
+++ b/docs/zh/12-taos-sql/02-database.md
@ -32,7 +32,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
   - cacheLast: [详细说明](/reference/config/#cachelast)
   - replica: [详细说明](/reference/config/#replica)
   - quorum: [详细说明](/reference/config/#quorum)
-   - maxVgroupsPerDb: [详细说明](/reference/config/#maxvgroupsperdb)
   - comp: [详细说明](/reference/config/#comp)
   - precision: [详细说明](/reference/config/#precision)
 6. 请注意上面列出的所有参数都可以配置在配置文件 `taosd.cfg` 中作为创建数据库时使用的默认配置， `create database` 的参数中明确指定的会覆盖配置文件中的设置。
--- a/examples/c/tmq.c
+++ b/examples/c/tmq.c
@ -28,15 +28,23 @@ static void msg_process(TAOS_RES* msg) {
  printf("db: %s\n", tmq_get_db_name(msg));
  printf("vg: %d\n", tmq_get_vgroup_id(msg));
  if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
-    void*   meta;
-    int32_t metaLen;
-    tmq_get_raw_meta(msg, &meta, &metaLen);
+    tmq_raw_data *raw = tmq_get_raw_meta(msg);
+    if(raw){
+      TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", "abc1", 0);
+      if (pConn == NULL) {
+        return;
+      }
+      int32_t ret = taos_write_raw_meta(pConn, raw);
+      printf("write raw data: %s\n", tmq_err2str(ret));
+      free(raw);
+      taos_close(pConn);
+    }
    char* result = tmq_get_json_meta(msg);
    if(result){
      printf("meta result: %s\n", result);
      free(result);
    }
-    printf("meta, len is %d\n", metaLen);
+    printf("meta:%p\n", raw);
    return;
  }
  while (1) {
--- a/include/client/taos.h
+++ b/include/client/taos.h
@ -260,15 +260,16 @@ enum tmq_res_t {
 };

 typedef enum tmq_res_t tmq_res_t;
+typedef struct tmq_raw_data tmq_raw_data;

-DLL_EXPORT tmq_res_t   tmq_get_res_type(TAOS_RES *res);
-DLL_EXPORT int32_t     tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len);
-DLL_EXPORT int32_t     taos_write_raw_meta(TAOS *res, void *raw_meta, int32_t raw_meta_len);
-DLL_EXPORT char       *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed.
-DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
-DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
-DLL_EXPORT int32_t     tmq_get_vgroup_id(TAOS_RES *res);
-DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res);
+DLL_EXPORT tmq_res_t     tmq_get_res_type(TAOS_RES *res);
+DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res);
+DLL_EXPORT int32_t       taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta);
+DLL_EXPORT char         *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed.
+DLL_EXPORT const char   *tmq_get_topic_name(TAOS_RES *res);
+DLL_EXPORT const char   *tmq_get_db_name(TAOS_RES *res);
+DLL_EXPORT int32_t       tmq_get_vgroup_id(TAOS_RES *res);
+DLL_EXPORT const char   *tmq_get_table_name(TAOS_RES *res);

 /* ------------------------------ TMQ END -------------------------------- */

--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@ -55,7 +55,8 @@ enum {
 enum {
  STREAM_INPUT__DATA_SUBMIT = 1,
  STREAM_INPUT__DATA_BLOCK,
-  STREAM_INPUT__DATA_SCAN,
+  // STREAM_INPUT__TABLE_SCAN,
+  STREAM_INPUT__TQ_SCAN,
  STREAM_INPUT__DATA_RETRIEVE,
  STREAM_INPUT__TRIGGER,
  STREAM_INPUT__CHECKPOINT,
@ -122,7 +123,8 @@ enum {
 };

 typedef struct {
-  int8_t fetchType;
+  int8_t       fetchType;
+  STqOffsetVal offset;
  union {
    SSDataBlock data;
    void*       meta;
@ -153,8 +155,8 @@ typedef struct SQueryTableDataCond {
  int32_t      numOfCols;
  SColumnInfo* colList;
  int32_t      type;  // data block load type:
-  int32_t      numOfTWindows;
-  STimeWindow* twindows;
+//  int32_t      numOfTWindows;
+  STimeWindow  twindows;
  int64_t      startVersion;
  int64_t      endVersion;
 } SQueryTableDataCond;
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@ -231,7 +231,7 @@ SSDataBlock* createDataBlock();
 int32_t      blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);

 SColumnInfoData  createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
-SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
+SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index);

 void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress);
 const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@ -67,7 +67,6 @@ extern int32_t tsNumOfVnodeQueryThreads;
 extern int32_t tsNumOfVnodeFetchThreads;
 extern int32_t tsNumOfVnodeWriteThreads;
 extern int32_t tsNumOfVnodeSyncThreads;
-extern int32_t tsNumOfVnodeMergeThreads;
 extern int32_t tsNumOfQnodeQueryThreads;
 extern int32_t tsNumOfQnodeFetchThreads;
 extern int32_t tsNumOfSnodeSharedThreads;
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@ -55,11 +55,11 @@ extern int32_t tMsgDict[];

 #define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
 #define TMSG_SEG_SEQ(TYPE)  ((TYPE)&0xff)
-#define TMSG_INFO(TYPE)                                                                                             \
-  ((TYPE) >= 0 &&                                                                                                   \
-   ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \
-    (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG))                         \
-      ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)]                                                \
+#define TMSG_INFO(TYPE)                                                                                      \
+  ((TYPE) >= 0 && ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG ||    \
+                   (TYPE) < TDMT_SCH_MAX_MSG || (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || \
+                   (TYPE) < TDMT_SYNC_MAX_MSG))                                                              \
+      ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)]                                         \
      : 0
 #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))

@ -169,6 +169,9 @@ typedef enum _mgmt_table {
 #define TD_CHILD_TABLE  TSDB_CHILD_TABLE
 #define TD_NORMAL_TABLE TSDB_NORMAL_TABLE

+#define TD_REQ_FROM_APP  0
+#define TD_REQ_FROM_TAOX 1
+
 typedef struct {
  int32_t vgId;
  char*   dbFName;
@ -432,25 +435,30 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
 STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);

 typedef struct {
-  char    name[TSDB_TABLE_FNAME_LEN];
-  int8_t  igExists;
-  int64_t delay1;
-  int64_t delay2;
-  int64_t watermark1;
-  int64_t watermark2;
-  int32_t ttl;
-  int32_t numOfColumns;
-  int32_t numOfTags;
-  int32_t numOfFuncs;
-  int32_t commentLen;
-  int32_t ast1Len;
-  int32_t ast2Len;
-  SArray* pColumns;  // array of SField
-  SArray* pTags;     // array of SField
-  SArray* pFuncs;
-  char*   pComment;
-  char*   pAst1;
-  char*   pAst2;
+  char     name[TSDB_TABLE_FNAME_LEN];
+  int8_t   igExists;
+  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   reserved[6];
+  tb_uid_t suid;
+  int64_t  delay1;
+  int64_t  delay2;
+  int64_t  watermark1;
+  int64_t  watermark2;
+  int32_t  ttl;
+  int32_t  colVer;
+  int32_t  tagVer;
+  int32_t  numOfColumns;
+  int32_t  numOfTags;
+  int32_t  numOfFuncs;
+  int32_t  commentLen;
+  int32_t  ast1Len;
+  int32_t  ast2Len;
+  SArray*  pColumns;  // array of SField
+  SArray*  pTags;     // array of SField
+  SArray*  pFuncs;
+  char*    pComment;
+  char*    pAst1;
+  char*    pAst2;
 } SMCreateStbReq;

 int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@ -458,8 +466,11 @@ int32_t tDeserializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pR
 void    tFreeSMCreateStbReq(SMCreateStbReq* pReq);

 typedef struct {
-  char   name[TSDB_TABLE_FNAME_LEN];
-  int8_t igNotExists;
+  char     name[TSDB_TABLE_FNAME_LEN];
+  int8_t   igNotExists;
+  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   reserved[6];
+  tb_uid_t suid;
 } SMDropStbReq;

 int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
@ -468,8 +479,6 @@ int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
 typedef struct {
  char    name[TSDB_TABLE_FNAME_LEN];
  int8_t  alterType;
-  int32_t tagVer;
-  int32_t colVer;
  int32_t numOfFields;
  SArray* pFields;
  int32_t ttl;
@ -714,7 +723,7 @@ typedef struct {
  int32_t buffer;        // MB
  int32_t pageSize;
  int32_t pages;
-  int32_t lastRowMem;
+  int32_t cacheLastSize;
  int32_t daysPerFile;
  int32_t daysToKeep0;
  int32_t daysToKeep1;
@ -727,7 +736,7 @@ typedef struct {
  int8_t  compression;
  int8_t  replications;
  int8_t  strict;
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int8_t  schemaless;
  int8_t  ignoreExist;
  int32_t numOfRetensions;
@ -743,7 +752,7 @@ typedef struct {
  int32_t buffer;
  int32_t pageSize;
  int32_t pages;
-  int32_t lastRowMem;
+  int32_t cacheLastSize;
  int32_t daysPerFile;
  int32_t daysToKeep0;
  int32_t daysToKeep1;
@ -751,7 +760,7 @@ typedef struct {
  int32_t fsyncPeriod;
  int8_t  walLevel;
  int8_t  strict;
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int8_t  replications;
 } SAlterDbReq;

@ -806,6 +815,13 @@ typedef struct {
 int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
 int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);

+typedef struct {
+  char db[TSDB_DB_FNAME_LEN];
+} STrimDbReq;
+
+int32_t tSerializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq);
+int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq);
+
 typedef struct {
  int32_t numOfVgroups;
  int32_t numOfStables;
@ -824,7 +840,7 @@ typedef struct {
  int8_t  compression;
  int8_t  replications;
  int8_t  strict;
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int32_t numOfRetensions;
  SArray* pRetensions;
  int8_t  schemaless;
@ -1089,7 +1105,7 @@ typedef struct {
  int32_t  buffer;
  int32_t  pageSize;
  int32_t  pages;
-  int32_t  lastRowMem;
+  int32_t  cacheLastSize;
  int32_t  daysPerFile;
  int32_t  daysToKeep0;
  int32_t  daysToKeep1;
@ -1104,7 +1120,7 @@ typedef struct {
  int8_t   precision;
  int8_t   compression;
  int8_t   strict;
-  int8_t   cacheLastRow;
+  int8_t   cacheLast;
  int8_t   isTsma;
  int8_t   standby;
  int8_t   replica;
@ -1142,7 +1158,7 @@ typedef struct {
  int32_t  buffer;
  int32_t  pageSize;
  int32_t  pages;
-  int32_t  lastRowMem;
+  int32_t  cacheLastSize;
  int32_t  daysPerFile;
  int32_t  daysToKeep0;
  int32_t  daysToKeep1;
@ -1150,7 +1166,7 @@ typedef struct {
  int32_t  fsyncPeriod;
  int8_t   walLevel;
  int8_t   strict;
-  int8_t   cacheLastRow;
+  int8_t   cacheLast;
  int8_t   selfIndex;
  int8_t   replica;
  SReplica replicas[TSDB_MAX_REPLICA];
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@ -116,6 +116,7 @@ enum {
  TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "alter-db", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "sync-db", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "compact-db", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB, "trim-db", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "get-db-cfg", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "vgroup-list", NULL, NULL)
  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "create-func", NULL, NULL)
--- a/include/common/tname.h
+++ b/include/common/tname.h
@ -37,6 +37,8 @@ typedef struct SName {
  char    tname[TSDB_TABLE_NAME_LEN];
 } SName;

+SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName);
+
 int32_t tNameExtractFullName(const SName* name, char* dst);

 int32_t tNameLen(const SName* name);
--- a/include/common/ttokendef.h
+++ b/include/common/ttokendef.h
@ -74,201 +74,202 @@
 #define TK_DATABASE       56
 #define TK_USE            57
 #define TK_FLUSH          58
-#define TK_IF             59
-#define TK_NOT            60
-#define TK_EXISTS         61
-#define TK_BUFFER         62
-#define TK_CACHELAST      63
-#define TK_CACHELASTSIZE  64
-#define TK_COMP           65
-#define TK_DURATION       66
-#define TK_NK_VARIABLE    67
-#define TK_FSYNC          68
-#define TK_MAXROWS        69
-#define TK_MINROWS        70
-#define TK_KEEP           71
-#define TK_PAGES          72
-#define TK_PAGESIZE       73
-#define TK_PRECISION      74
-#define TK_REPLICA        75
-#define TK_STRICT         76
-#define TK_WAL            77
-#define TK_VGROUPS        78
-#define TK_SINGLE_STABLE  79
-#define TK_RETENTIONS     80
-#define TK_SCHEMALESS     81
-#define TK_NK_COLON       82
-#define TK_TABLE          83
-#define TK_NK_LP          84
-#define TK_NK_RP          85
-#define TK_STABLE         86
-#define TK_ADD            87
-#define TK_COLUMN         88
-#define TK_MODIFY         89
-#define TK_RENAME         90
-#define TK_TAG            91
-#define TK_SET            92
-#define TK_NK_EQ          93
-#define TK_USING          94
-#define TK_TAGS           95
-#define TK_COMMENT        96
-#define TK_BOOL           97
-#define TK_TINYINT        98
-#define TK_SMALLINT       99
-#define TK_INT            100
-#define TK_INTEGER        101
-#define TK_BIGINT         102
-#define TK_FLOAT          103
-#define TK_DOUBLE         104
-#define TK_BINARY         105
-#define TK_TIMESTAMP      106
-#define TK_NCHAR          107
-#define TK_UNSIGNED       108
-#define TK_JSON           109
-#define TK_VARCHAR        110
-#define TK_MEDIUMBLOB     111
-#define TK_BLOB           112
-#define TK_VARBINARY      113
-#define TK_DECIMAL        114
-#define TK_MAX_DELAY      115
-#define TK_WATERMARK      116
-#define TK_ROLLUP         117
-#define TK_TTL            118
-#define TK_SMA            119
-#define TK_FIRST          120
-#define TK_LAST           121
-#define TK_SHOW           122
-#define TK_DATABASES      123
-#define TK_TABLES         124
-#define TK_STABLES        125
-#define TK_MNODES         126
-#define TK_MODULES        127
-#define TK_QNODES         128
-#define TK_FUNCTIONS      129
-#define TK_INDEXES        130
-#define TK_ACCOUNTS       131
-#define TK_APPS           132
-#define TK_CONNECTIONS    133
-#define TK_LICENCE        134
-#define TK_GRANTS         135
-#define TK_QUERIES        136
-#define TK_SCORES         137
-#define TK_TOPICS         138
-#define TK_VARIABLES      139
-#define TK_BNODES         140
-#define TK_SNODES         141
-#define TK_CLUSTER        142
-#define TK_TRANSACTIONS   143
-#define TK_DISTRIBUTED    144
-#define TK_CONSUMERS      145
-#define TK_SUBSCRIPTIONS  146
-#define TK_LIKE           147
-#define TK_INDEX          148
-#define TK_FUNCTION       149
-#define TK_INTERVAL       150
-#define TK_TOPIC          151
-#define TK_AS             152
-#define TK_WITH           153
-#define TK_META           154
-#define TK_CONSUMER       155
-#define TK_GROUP          156
-#define TK_DESC           157
-#define TK_DESCRIBE       158
-#define TK_RESET          159
-#define TK_QUERY          160
-#define TK_CACHE          161
-#define TK_EXPLAIN        162
-#define TK_ANALYZE        163
-#define TK_VERBOSE        164
-#define TK_NK_BOOL        165
-#define TK_RATIO          166
-#define TK_NK_FLOAT       167
-#define TK_COMPACT        168
-#define TK_VNODES         169
-#define TK_IN             170
-#define TK_OUTPUTTYPE     171
-#define TK_AGGREGATE      172
-#define TK_BUFSIZE        173
-#define TK_STREAM         174
-#define TK_INTO           175
-#define TK_TRIGGER        176
-#define TK_AT_ONCE        177
-#define TK_WINDOW_CLOSE   178
-#define TK_IGNORE         179
-#define TK_EXPIRED        180
-#define TK_KILL           181
-#define TK_CONNECTION     182
-#define TK_TRANSACTION    183
-#define TK_BALANCE        184
-#define TK_VGROUP         185
-#define TK_MERGE          186
-#define TK_REDISTRIBUTE   187
-#define TK_SPLIT          188
-#define TK_SYNCDB         189
-#define TK_DELETE         190
-#define TK_INSERT         191
-#define TK_NULL           192
-#define TK_NK_QUESTION    193
-#define TK_NK_ARROW       194
-#define TK_ROWTS          195
-#define TK_TBNAME         196
-#define TK_QSTARTTS       197
-#define TK_QENDTS         198
-#define TK_WSTARTTS       199
-#define TK_WENDTS         200
-#define TK_WDURATION      201
-#define TK_CAST           202
-#define TK_NOW            203
-#define TK_TODAY          204
-#define TK_TIMEZONE       205
-#define TK_CLIENT_VERSION 206
-#define TK_SERVER_VERSION 207
-#define TK_SERVER_STATUS  208
-#define TK_CURRENT_USER   209
-#define TK_COUNT          210
-#define TK_LAST_ROW       211
-#define TK_BETWEEN        212
-#define TK_IS             213
-#define TK_NK_LT          214
-#define TK_NK_GT          215
-#define TK_NK_LE          216
-#define TK_NK_GE          217
-#define TK_NK_NE          218
-#define TK_MATCH          219
-#define TK_NMATCH         220
-#define TK_CONTAINS       221
-#define TK_JOIN           222
-#define TK_INNER          223
-#define TK_SELECT         224
-#define TK_DISTINCT       225
-#define TK_WHERE          226
-#define TK_PARTITION      227
-#define TK_BY             228
-#define TK_SESSION        229
-#define TK_STATE_WINDOW   230
-#define TK_SLIDING        231
-#define TK_FILL           232
-#define TK_VALUE          233
-#define TK_NONE           234
-#define TK_PREV           235
-#define TK_LINEAR         236
-#define TK_NEXT           237
-#define TK_HAVING         238
-#define TK_RANGE          239
-#define TK_EVERY          240
-#define TK_ORDER          241
-#define TK_SLIMIT         242
-#define TK_SOFFSET        243
-#define TK_LIMIT          244
-#define TK_OFFSET         245
-#define TK_ASC            246
-#define TK_NULLS          247
-#define TK_ID             248
-#define TK_NK_BITNOT      249
-#define TK_VALUES         250
-#define TK_IMPORT         251
-#define TK_NK_SEMI        252
-#define TK_FILE           253
+#define TK_TRIM           59
+#define TK_IF             60
+#define TK_NOT            61
+#define TK_EXISTS         62
+#define TK_BUFFER         63
+#define TK_CACHELAST      64
+#define TK_CACHELASTSIZE  65
+#define TK_COMP           66
+#define TK_DURATION       67
+#define TK_NK_VARIABLE    68
+#define TK_FSYNC          69
+#define TK_MAXROWS        70
+#define TK_MINROWS        71
+#define TK_KEEP           72
+#define TK_PAGES          73
+#define TK_PAGESIZE       74
+#define TK_PRECISION      75
+#define TK_REPLICA        76
+#define TK_STRICT         77
+#define TK_WAL            78
+#define TK_VGROUPS        79
+#define TK_SINGLE_STABLE  80
+#define TK_RETENTIONS     81
+#define TK_SCHEMALESS     82
+#define TK_NK_COLON       83
+#define TK_TABLE          84
+#define TK_NK_LP          85
+#define TK_NK_RP          86
+#define TK_STABLE         87
+#define TK_ADD            88
+#define TK_COLUMN         89
+#define TK_MODIFY         90
+#define TK_RENAME         91
+#define TK_TAG            92
+#define TK_SET            93
+#define TK_NK_EQ          94
+#define TK_USING          95
+#define TK_TAGS           96
+#define TK_COMMENT        97
+#define TK_BOOL           98
+#define TK_TINYINT        99
+#define TK_SMALLINT       100
+#define TK_INT            101
+#define TK_INTEGER        102
+#define TK_BIGINT         103
+#define TK_FLOAT          104
+#define TK_DOUBLE         105
+#define TK_BINARY         106
+#define TK_TIMESTAMP      107
+#define TK_NCHAR          108
+#define TK_UNSIGNED       109
+#define TK_JSON           110
+#define TK_VARCHAR        111
+#define TK_MEDIUMBLOB     112
+#define TK_BLOB           113
+#define TK_VARBINARY      114
+#define TK_DECIMAL        115
+#define TK_MAX_DELAY      116
+#define TK_WATERMARK      117
+#define TK_ROLLUP         118
+#define TK_TTL            119
+#define TK_SMA            120
+#define TK_FIRST          121
+#define TK_LAST           122
+#define TK_SHOW           123
+#define TK_DATABASES      124
+#define TK_TABLES         125
+#define TK_STABLES        126
+#define TK_MNODES         127
+#define TK_MODULES        128
+#define TK_QNODES         129
+#define TK_FUNCTIONS      130
+#define TK_INDEXES        131
+#define TK_ACCOUNTS       132
+#define TK_APPS           133
+#define TK_CONNECTIONS    134
+#define TK_LICENCE        135
+#define TK_GRANTS         136
+#define TK_QUERIES        137
+#define TK_SCORES         138
+#define TK_TOPICS         139
+#define TK_VARIABLES      140
+#define TK_BNODES         141
+#define TK_SNODES         142
+#define TK_CLUSTER        143
+#define TK_TRANSACTIONS   144
+#define TK_DISTRIBUTED    145
+#define TK_CONSUMERS      146
+#define TK_SUBSCRIPTIONS  147
+#define TK_LIKE           148
+#define TK_INDEX          149
+#define TK_FUNCTION       150
+#define TK_INTERVAL       151
+#define TK_TOPIC          152
+#define TK_AS             153
+#define TK_WITH           154
+#define TK_META           155
+#define TK_CONSUMER       156
+#define TK_GROUP          157
+#define TK_DESC           158
+#define TK_DESCRIBE       159
+#define TK_RESET          160
+#define TK_QUERY          161
+#define TK_CACHE          162
+#define TK_EXPLAIN        163
+#define TK_ANALYZE        164
+#define TK_VERBOSE        165
+#define TK_NK_BOOL        166
+#define TK_RATIO          167
+#define TK_NK_FLOAT       168
+#define TK_COMPACT        169
+#define TK_VNODES         170
+#define TK_IN             171
+#define TK_OUTPUTTYPE     172
+#define TK_AGGREGATE      173
+#define TK_BUFSIZE        174
+#define TK_STREAM         175
+#define TK_INTO           176
+#define TK_TRIGGER        177
+#define TK_AT_ONCE        178
+#define TK_WINDOW_CLOSE   179
+#define TK_IGNORE         180
+#define TK_EXPIRED        181
+#define TK_KILL           182
+#define TK_CONNECTION     183
+#define TK_TRANSACTION    184
+#define TK_BALANCE        185
+#define TK_VGROUP         186
+#define TK_MERGE          187
+#define TK_REDISTRIBUTE   188
+#define TK_SPLIT          189
+#define TK_SYNCDB         190
+#define TK_DELETE         191
+#define TK_INSERT         192
+#define TK_NULL           193
+#define TK_NK_QUESTION    194
+#define TK_NK_ARROW       195
+#define TK_ROWTS          196
+#define TK_TBNAME         197
+#define TK_QSTARTTS       198
+#define TK_QENDTS         199
+#define TK_WSTARTTS       200
+#define TK_WENDTS         201
+#define TK_WDURATION      202
+#define TK_CAST           203
+#define TK_NOW            204
+#define TK_TODAY          205
+#define TK_TIMEZONE       206
+#define TK_CLIENT_VERSION 207
+#define TK_SERVER_VERSION 208
+#define TK_SERVER_STATUS  209
+#define TK_CURRENT_USER   210
+#define TK_COUNT          211
+#define TK_LAST_ROW       212
+#define TK_BETWEEN        213
+#define TK_IS             214
+#define TK_NK_LT          215
+#define TK_NK_GT          216
+#define TK_NK_LE          217
+#define TK_NK_GE          218
+#define TK_NK_NE          219
+#define TK_MATCH          220
+#define TK_NMATCH         221
+#define TK_CONTAINS       222
+#define TK_JOIN           223
+#define TK_INNER          224
+#define TK_SELECT         225
+#define TK_DISTINCT       226
+#define TK_WHERE          227
+#define TK_PARTITION      228
+#define TK_BY             229
+#define TK_SESSION        230
+#define TK_STATE_WINDOW   231
+#define TK_SLIDING        232
+#define TK_FILL           233
+#define TK_VALUE          234
+#define TK_NONE           235
+#define TK_PREV           236
+#define TK_LINEAR         237
+#define TK_NEXT           238
+#define TK_HAVING         239
+#define TK_RANGE          240
+#define TK_EVERY          241
+#define TK_ORDER          242
+#define TK_SLIMIT         243
+#define TK_SOFFSET        244
+#define TK_LIMIT          245
+#define TK_OFFSET         246
+#define TK_ASC            247
+#define TK_NULLS          248
+#define TK_ID             249
+#define TK_NK_BITNOT      250
+#define TK_VALUES         251
+#define TK_IMPORT         252
+#define TK_NK_SEMI        253
+#define TK_FILE           254

 #define TK_NK_SPACE   300
 #define TK_NK_COMMENT 301
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@ -36,6 +36,7 @@ typedef struct SReadHandle {
  void*   vnode;
  void*   mnd;
  SMsgCb* pMsgCb;
+  int64_t version;
  bool    initMetaReader;
  bool    initTableReader;
  bool    initTqReader;
@ -109,7 +110,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
 * @param tversion
 * @return
 */
-int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
+int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
                                      int32_t* tversion);

 /**
@ -174,7 +175,13 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
 */
 int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);

-int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
+int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
+
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset);
+
+int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);
+
+void* qStreamExtractMetaMsg(qTaskInfo_t tinfo);

 void*   qExtractReaderFromStreamScanner(void* scanner);
 int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@ -172,7 +172,13 @@ typedef struct tExprNode {

 void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));

+typedef enum {
+  SHOULD_FREE_COLDATA    = 0x1,   // the newly created column data needs to be destroyed.
+  DELEGATED_MGMT_COLDATA = 0x2,   // input column data should not be released.
+} ECOLDATA_MGMT_TYPE_E;
+
 struct SScalarParam {
+  ECOLDATA_MGMT_TYPE_E type;
  SColumnInfoData *columnData;
  SHashObj        *pHashFilter;
  int32_t          hashValueType;
--- a/include/libs/nodes/cmdnodes.h
+++ b/include/libs/nodes/cmdnodes.h
@ -103,6 +103,11 @@ typedef struct SFlushDatabaseStmt {
  char      dbName[TSDB_DB_NAME_LEN];
 } SFlushDatabaseStmt;

+typedef struct STrimDatabaseStmt {
+  ENodeType type;
+  char      dbName[TSDB_DB_NAME_LEN];
+} STrimDatabaseStmt;
+
 typedef struct STableOptions {
  ENodeType  type;
  bool       commentNull;
--- a/include/libs/nodes/nodes.h
+++ b/include/libs/nodes/nodes.h
@ -112,6 +112,7 @@ typedef enum ENodeType {
  QUERY_NODE_DROP_DATABASE_STMT,
  QUERY_NODE_ALTER_DATABASE_STMT,
  QUERY_NODE_FLUSH_DATABASE_STMT,
+  QUERY_NODE_TRIM_DATABASE_STMT,
  QUERY_NODE_CREATE_TABLE_STMT,
  QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
  QUERY_NODE_CREATE_MULTI_TABLE_STMT,
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@ -250,6 +250,7 @@ typedef struct SSelectStmt {
  SLimitNode* pSlimit;
  char        stmtName[TSDB_TABLE_NAME_LEN];
  uint8_t     precision;
+  int32_t     selectFuncNum;
  bool        isEmptyResult;
  bool        isTimeLineResult;
  bool        hasAggFuncs;
@ -257,6 +258,7 @@ typedef struct SSelectStmt {
  bool        hasIndefiniteRowsFunc;
  bool        hasSelectFunc;
  bool        hasSelectValFunc;
+  bool        hasOtherVectorFunc;
  bool        hasUniqueFunc;
  bool        hasTailFunc;
  bool        hasInterpFunc;
--- a/include/libs/parser/parser.h
+++ b/include/libs/parser/parser.h
@ -96,6 +96,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
                    char* tableName, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);

+int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
+SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
+SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
 #ifdef __cplusplus
 }
 #endif
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@ -25,7 +25,7 @@ extern "C" {

 typedef struct SFilterInfo SFilterInfo;

-int32_t scalarGetOperatorResultType(SDataType left, SDataType right, EOperatorType op, SDataType* pRes);
+int32_t scalarGetOperatorResultType(SOperatorNode* pOp);

 /*
 pNode will be freed in API;
@ -42,7 +42,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type);
 int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);

 int32_t vectorGetConvertType(int32_t type1, int32_t type2);
-int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut);
+int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* overflow);

 /* Math functions */
 int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@ -27,7 +27,7 @@ extern "C" {

 #define TAOS_CONN_SERVER 0
 #define TAOS_CONN_CLIENT 1
-#define IsReq(pMsg) (pMsg->msgType & 1U)
+#define IsReq(pMsg)      (pMsg->msgType & 1U)

 extern int32_t tsRpcHeadSize;

--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@ -24,42 +24,14 @@
 extern "C" {
 #endif

-#define wFatal(...)                                              \
-  {                                                              \
-    if (wDebugFlag & DEBUG_FATAL) {                              \
-      taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
-    }                                                            \
-  }
-#define wError(...)                                              \
-  {                                                              \
-    if (wDebugFlag & DEBUG_ERROR) {                              \
-      taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
-    }                                                            \
-  }
-#define wWarn(...)                                             \
-  {                                                            \
-    if (wDebugFlag & DEBUG_WARN) {                             \
-      taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
-    }                                                          \
-  }
-#define wInfo(...)                                        \
-  {                                                       \
-    if (wDebugFlag & DEBUG_INFO) {                        \
-      taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
-    }                                                     \
-  }
-#define wDebug(...)                                               \
-  {                                                               \
-    if (wDebugFlag & DEBUG_DEBUG) {                               \
-      taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
-    }                                                             \
-  }
-#define wTrace(...)                                               \
-  {                                                               \
-    if (wDebugFlag & DEBUG_TRACE) {                               \
-      taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
-    }                                                             \
-  }
+// clang-format off
+#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255,        __VA_ARGS__); }}
+#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255,        __VA_ARGS__); }}
+#define wWarn(...)  { if (wDebugFlag & DEBUG_WARN)  { taosPrintLog("WAL WARN ",  DEBUG_WARN, 255,         __VA_ARGS__); }}
+#define wInfo(...)  { if (wDebugFlag & DEBUG_INFO)  { taosPrintLog("WAL ",       DEBUG_INFO, 255,         __VA_ARGS__); }}
+#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ",       DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }}
+#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ",       DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }}
+// clang-format on

 #define WAL_PROTO_VER    0
 #define WAL_NOSUFFIX_LEN 20
@ -73,7 +45,6 @@ extern "C" {
 #define WAL_MAGIC        0xFAFBFCFDULL

 typedef enum {
-  TAOS_WAL_NOLOG = 0,
  TAOS_WAL_WRITE = 1,
  TAOS_WAL_FSYNC = 2,
 } EWalType;
@ -102,7 +73,7 @@ typedef struct {
  int8_t   isWeek;
  uint64_t seqNum;
  uint64_t term;
-} SSyncLogMeta;
+} SWalSyncInfo;

 typedef struct {
  int8_t  protoVer;
@ -112,7 +83,7 @@ typedef struct {
  int64_t ingestTs;  // not implemented

  // sync meta
-  SSyncLogMeta syncMeta;
+  SWalSyncInfo syncMeta;

  char body[];
 } SWalCont;
@ -152,6 +123,7 @@ typedef struct SWal {
 typedef struct {
  int8_t scanUncommited;
  int8_t scanMeta;
+  int8_t enableRef;
 } SWalFilterCond;

 typedef struct {
@ -161,6 +133,7 @@ typedef struct {
  int64_t        curFileFirstVer;
  int64_t        curVersion;
  int64_t        capacity;
+  int8_t         curInvalid;
  TdThreadMutex  mutex;
  SWalFilterCond cond;
  SWalCkHead    *pHead;
@ -175,11 +148,22 @@ SWal   *walOpen(const char *path, SWalCfg *pCfg);
 int32_t walAlter(SWal *, SWalCfg *pCfg);
 void    walClose(SWal *);

-// write
-int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body,
-                             int32_t bodyLen);
+// write interfaces
+
+// By assigning index by the caller, wal gurantees linearizability
 int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
-void    walFsync(SWal *, bool force);
+int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
+                             int32_t bodyLen);
+
+// This interface assign version automatically and return to caller.
+// When using this interface with concurrent writes,
+// wal will write all logs atomically,
+// but not sure which one will be actually write first,
+// and then the unique index of successful writen is returned.
+// -1 will be returned for failed writes
+int64_t walAppendLog(SWal *, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
+
+void walFsync(SWal *, bool force);

 // apis for lifecycle management
 int32_t walCommit(SWal *, int64_t ver);
@ -194,6 +178,7 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
 SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond);
 void        walCloseReader(SWalReader *pRead);
 int32_t     walReadVer(SWalReader *pRead, int64_t ver);
+int32_t     walReadSeekVer(SWalReader *pRead, int64_t ver);
 int32_t     walNextValidMsg(SWalReader *pRead);

 // only for tq usage
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@ -247,9 +247,11 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
 #define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF)
 #define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
+#define TSDB_CODE_MND_INVALID_SCHEMA_VER        TAOS_DEF_ERROR_CODE(0, 0x03B1)
+#define TSDB_CODE_MND_STABLE_UID_NOT_MATCH      TAOS_DEF_ERROR_CODE(0, 0x03B2)

 // mnode-infoSchema
-#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03BA)
+#define TSDB_CODE_MND_INVALID_SYS_TABLENAME     TAOS_DEF_ERROR_CODE(0, 0x03BA)

 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)
@ -582,6 +584,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_INTERP_CLAUSE     TAOS_DEF_ERROR_CODE(0, 0x265D)
 #define TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN      TAOS_DEF_ERROR_CODE(0, 0x265E)
 #define TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE TAOS_DEF_ERROR_CODE(0, 0x265F)
+#define TSDB_CODE_PAR_INVALID_SMA_INDEX         TAOS_DEF_ERROR_CODE(0, 0x265C)

 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@ -331,12 +331,12 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_STRICT_OFF              0
 #define TSDB_DB_STRICT_ON               1
 #define TSDB_DEFAULT_DB_STRICT          0
-#define TSDB_MIN_DB_CACHE_LAST_ROW      0
-#define TSDB_MAX_DB_CACHE_LAST_ROW      3
-#define TSDB_DEFAULT_CACHE_LAST_ROW     0
-#define TSDB_MIN_DB_LAST_ROW_MEM        1  // MB
-#define TSDB_MAX_DB_LAST_ROW_MEM        65536
-#define TSDB_DEFAULT_LAST_ROW_MEM       1
+#define TSDB_MIN_DB_CACHE_LAST          0
+#define TSDB_MAX_DB_CACHE_LAST          3
+#define TSDB_DEFAULT_CACHE_LAST         0
+#define TSDB_MIN_DB_CACHE_LAST_SIZE     1  // MB
+#define TSDB_MAX_DB_CACHE_LAST_SIZE     65536
+#define TSDB_DEFAULT_CACHE_LAST_SIZE    1
 #define TSDB_DB_STREAM_MODE_OFF         0
 #define TSDB_DB_STREAM_MODE_ON          1
 #define TSDB_DEFAULT_DB_STREAM_MODE     0
--- a/include/util/tlockfree.h
+++ b/include/util/tlockfree.h
@ -69,13 +69,14 @@ typedef void (*_ref_fn_t)(const void *pObj);
 #define T_REF_VAL_GET(x) (x)->_ref.val

 // single writer multiple reader lock
-typedef volatile int32_t SRWLatch;
+typedef volatile int64_t SRWLatch;

-void taosInitRWLatch(SRWLatch *pLatch);
-void taosWLockLatch(SRWLatch *pLatch);
-void taosWUnLockLatch(SRWLatch *pLatch);
-void taosRLockLatch(SRWLatch *pLatch);
-void taosRUnLockLatch(SRWLatch *pLatch);
+void    taosInitRWLatch(SRWLatch *pLatch);
+void    taosInitReentrantRWLatch(SRWLatch *pLatch);
+void    taosWLockLatch(SRWLatch *pLatch);
+void    taosWUnLockLatch(SRWLatch *pLatch);
+void    taosRLockLatch(SRWLatch *pLatch);
+void    taosRUnLockLatch(SRWLatch *pLatch);
 int32_t taosWTryLockLatch(SRWLatch *pLatch);

 // copy on read
--- a/include/util/tlog.h
+++ b/include/util/tlog.h
@ -94,7 +94,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
 // clang-format on
-
+#define BUF_PAGE_DEBUG
 #ifdef __cplusplus
 }
 #endif
--- a/packaging/tools/install.sh
+++ b/packaging/tools/install.sh
@ -30,6 +30,7 @@ configDir="/etc/taos"
 installDir="/usr/local/taos"
 adapterName="taosadapter"
 benchmarkName="taosBenchmark"
+tmqName="tmq_sim"
 dumpName="taosdump"
 demoName="taosdemo"

@ -205,6 +206,7 @@ function install_bin() {
  [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
  [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
  [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
+  [ -x ${install_main_dir}/bin/${tmqName} ] && ${csudo}ln -s ${install_main_dir}/bin/${tmqName} ${bin_link_dir}/${tmqName} || :
  [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
  [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
  [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
--- a/packaging/tools/make_install.sh
+++ b/packaging/tools/make_install.sh
@ -305,18 +305,23 @@ function install_lib() {
      ${install_main_dir}/driver &&
      ${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber}

-    ${csudo}cp ${binary_dir}/build/lib/libtaosws.so \
-      ${install_main_dir}/driver &&
-      ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so
-
    ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
    ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
-    ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
-
    if [ -d "${lib64_link_dir}" ]; then
-      ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1
-      ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so
-      ${csudo}ln -sf ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
+        ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1
+        ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so
+    fi
+
+    if [ -f ${binary_dir}/build/lib/libtaosws.so ]; then
+        ${csudo}cp ${binary_dir}/build/lib/libtaosws.so \
+            ${install_main_dir}/driver &&
+            ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||:
+
+        ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
+
+        if [ -d "${lib64_link_dir}" ]; then
+            ${csudo}ln -sf ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
+        fi
    fi
  else
    ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \
@ -357,26 +362,26 @@ function install_header() {

  if [ "$osType" != "Darwin" ]; then
    ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
+    ${csudo}rm -f ${inc_link_dir}/taosws.h ||:
    ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
-    ${csudo}rm -f ${inc_link_dir}/taosws.h || :
-
-    ${csudo}cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taosdef.h ${source_dir}/src/inc/taoserror.h \
      ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*

-    ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h
+    if [ -f ${binary_dir}/build/include/taosws.h ]; then
+      ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||:
+      ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||:
+    fi

    ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
    ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
    ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
    ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h

-    ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || :
  else
    ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
      ${install_main_dir}/include ||
      ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
        ${install_main_2_dir}/include &&
-      ${csudo}chmod 644 ${install_main_dir}/include/* ||
+      ${csudo}chmod 644 ${install_main_dir}/include/* ||:
      ${csudo}chmod 644 ${install_main_2_dir}/include/*
  fi
 }
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@ -60,7 +60,7 @@ if [ "$pagMode" == "lite" ]; then
  strip ${build_dir}/bin/${serverName}
  strip ${build_dir}/bin/${clientName}
  # lite version doesn't include taosadapter,  which will lead to no restful interface
-  bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark"
+  bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark  ${build_dir}/bin/tmq_sim"
  taostools_bin_files=""
 else

@ -78,6 +78,7 @@ else

  taostools_bin_files=" ${build_dir}/bin/taosdump \
      ${build_dir}/bin/taosBenchmark \
+      ${build_dir}/bin/tmq_sim \
      ${build_dir}/bin/TDinsight.sh \
      $tdinsight_caches"

--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@ -222,8 +222,8 @@ typedef struct SRequestObj {
  SArray*              tableList;
  SQueryExecMetric     metric;
  SRequestSendRecvBody body;
-  bool                 stableQuery;
-  bool                 validateOnly;
+  bool                 stableQuery;  // todo refactor
+  bool                 validateOnly; // todo refactor

  bool     killed;
  uint32_t prevCode;  // previous error code: todo refactor, add update flag for catalog
@ -247,9 +247,9 @@ void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
 int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
 void    syncCatalogFn(SMetaData* pResult, void* param, int32_t code);

-SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly);
+SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly);
 TAOS_RES*    taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
-void         taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
+void         taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);

 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
  SMqRspObj* msg = (SMqRspObj*)res;
@ -297,7 +297,7 @@ int32_t  releaseTscObj(int64_t rid);

 uint64_t generateRequestId();

-void*        createRequest(STscObj* pObj, int32_t type);
+void*        createRequest(uint64_t connId, int32_t type);
 void         destroyRequest(SRequestObj* pRequest);
 SRequestObj* acquireRequest(int64_t rid);
 int32_t      releaseRequest(int64_t rid);
@ -318,13 +318,13 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
 STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                               uint16_t port, int connType);

-SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly);
+SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly);

 int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);

 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);

-int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
+int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql, SRequestObj** pRequest);

 void taos_close_internal(void* taos);

--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@ -49,13 +49,8 @@ void cleanupTscQhandle() {
  // destroy handle
  taosCleanUpScheduler(tscQhandle);
 }
-static int32_t registerRequest(SRequestObj *pRequest) {
-  STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id);
-  if (NULL == pTscObj) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return terrno;
-  }

+static int32_t registerRequest(SRequestObj *pRequest, STscObj* pTscObj) {
  // connection has been released already, abort creating request.
  pRequest->self = taosAddRef(clientReqRefPool, pRequest);

@ -246,29 +241,34 @@ STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientCon

 int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); }

-void *createRequest(STscObj *pObj, int32_t type) {
-  assert(pObj != NULL);
-
+void *createRequest(uint64_t connId, int32_t type) {
  SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj));
  if (NULL == pRequest) {
    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
    return NULL;
  }

+  STscObj* pTscObj = acquireTscObj(connId);
+  if (pTscObj == NULL) {
+    terrno = TSDB_CODE_TSC_DISCONNECTED;
+    return NULL;
+  }
+
  pRequest->resType = RES_TYPE__QUERY;
-  pRequest->pDb = getDbOfConnection(pObj);
  pRequest->requestId = generateRequestId();
  pRequest->metric.start = taosGetTimestampUs();

  pRequest->body.resInfo.convertUcs4 = true;  // convert ucs4 by default
-
  pRequest->type = type;
-  pRequest->pTscObj = pObj;
+
+  pRequest->pDb = getDbOfConnection(pTscObj);
+  pRequest->pTscObj = pTscObj;
+
  pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
  pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE;
  tsem_init(&pRequest->body.rspSem, 0, 0);

-  if (registerRequest(pRequest)) {
+  if (registerRequest(pRequest, pTscObj)) {
    doDestroyRequest(pRequest);
    return NULL;
  }
@ -327,8 +327,8 @@ void doDestroyRequest(void *p) {
  if (pRequest->self) {
    deregisterRequest(pRequest);
  }
-  taosMemoryFree(pRequest);

+  taosMemoryFree(pRequest);
  tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest);
 }

@ -338,7 +338,6 @@ void destroyRequest(SRequestObj *pRequest) {
  }

  taos_stop_query(pRequest);
-
  removeRequest(pRequest->self);
 }

--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@ -264,15 +264,12 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {

 static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
  static int32_t emptyRspNum = 0;
-  if (code != 0) {
-    taosMemoryFreeClear(param);
-    return -1;
-  }
-
  char             *key = (char *)param;
  SClientHbBatchRsp pRsp = {0};
-  tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
-
+  if (TSDB_CODE_SUCCESS == code) {
+    tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
+  }
+  
  int32_t rspNum = taosArrayGetSize(pRsp.rsps);

  taosThreadMutexLock(&appInfo.mutex);
@ -288,6 +285,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {

  taosMemoryFreeClear(param);

+  if (code != 0) {
+    (*pInst)->onlineDnodes = 0;
+  }
+
  if (rspNum) {
    tscDebug("hb got %d rsp, %d empty rsp received before", rspNum,
             atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0));
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@ -148,29 +148,50 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
  return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
 }

-int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) {
-  *pRequest = createRequest(pTscObj, TSDB_SQL_SELECT);
+int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql,
+                     SRequestObj** pRequest) {
+  *pRequest = createRequest(connId, TSDB_SQL_SELECT);
  if (*pRequest == NULL) {
-    tscError("failed to malloc sqlObj");
+    tscError("failed to malloc sqlObj, %s", sql);
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

  (*pRequest)->sqlstr = taosMemoryMalloc(sqlLen + 1);
  if ((*pRequest)->sqlstr == NULL) {
-    tscError("0x%" PRIx64 " failed to prepare sql string buffer", (*pRequest)->self);
-    (*pRequest)->msgBuf = strdup("failed to prepare sql string buffer");
+    tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql);
+    destroyRequest(*pRequest);
+    *pRequest = NULL;
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

  strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen);
  (*pRequest)->sqlstr[sqlLen] = 0;
  (*pRequest)->sqlLen = sqlLen;
+  (*pRequest)->validateOnly = validateSql;

+  if (param == NULL) {
+    SSyncQueryParam* pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+    if (pParam == NULL) {
+      destroyRequest(*pRequest);
+      *pRequest = NULL;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+
+    tsem_init(&pParam->sem, 0, 0);
+    pParam->pRequest = (*pRequest);
+    param = pParam;
+  }
+
+  (*pRequest)->body.param = param;
+
+  STscObj* pTscObj = (*pRequest)->pTscObj;
  if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
                  sizeof((*pRequest)->self))) {
+    tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self,
+             (*pRequest)->requestId, pTscObj->id, sql);
+
    destroyRequest(*pRequest);
    *pRequest = NULL;
-    tscError("put request to request hash failed");
    return TSDB_CODE_TSC_OUT_OF_MEMORY;
  }

@ -325,11 +346,14 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
  if (pInfo->pQnodeList) {
    taosArrayDestroy(pInfo->pQnodeList);
    pInfo->pQnodeList = NULL;
+    tscDebug("QnodeList cleared in cluster 0x%" PRIx64, pInfo->clusterId);
  }

  if (pNodeList) {
    pInfo->pQnodeList = taosArrayDup(pNodeList);
    taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad);
+    tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId,
+             taosArrayGetSize(pInfo->pQnodeList));
  }
  taosThreadMutexUnlock(&pInfo->qnodeMutex);

@ -627,22 +651,22 @@ _return:
 int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
  void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;

-  SExecResult     res = {0};
+  SExecResult      res = {0};
  SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
                           .requestId = pRequest->requestId,
                           .requestObjRefId = pRequest->self};
  SSchedulerReq    req = {
-    .syncReq = true,
-    .pConn = &conn,
-    .pNodeList = pNodeList,
-    .pDag = pDag,
-    .sql = pRequest->sqlstr,
-    .startTs = pRequest->metric.start,
-    .execFp = NULL,
-    .cbParam = NULL,
-    .chkKillFp = chkRequestKilled,
-    .chkKillParam = (void*)pRequest->self,
-    .pExecRes = &res,
+         .syncReq = true,
+         .pConn = &conn,
+         .pNodeList = pNodeList,
+         .pDag = pDag,
+         .sql = pRequest->sqlstr,
+         .startTs = pRequest->metric.start,
+         .execFp = NULL,
+         .cbParam = NULL,
+         .chkKillFp = chkRequestKilled,
+         .chkKillParam = (void*)pRequest->self,
+         .pExecRes = &res,
  };

  int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob);
@ -756,7 +780,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
    return code;
  }

-  SEpSet         epset = getEpSet_s(&pAppInfo->mgmtEp);
+  SEpSet       epset = getEpSet_s(&pAppInfo->mgmtEp);
  SExecResult* pRes = &pRequest->body.resInfo.execRes;

  switch (pRes->msgType) {
@ -786,11 +810,16 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
 void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
  SRequestObj* pRequest = (SRequestObj*)param;
  pRequest->code = code;
-  memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult));
+
+  if (pResult) {
+    memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult));
+  }

  if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type ||
      TDMT_VND_CREATE_TABLE == pRequest->type) {
-    pRequest->body.resInfo.numOfRows = pResult->numOfRows;
+    if (pResult) {
+      pRequest->body.resInfo.numOfRows = pResult->numOfRows;
+    }

    schedulerFreeJob(&pRequest->body.queryJob, 0);
  }
@ -880,18 +909,16 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
  return pRequest;
 }

-SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) {
+SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) {
  SRequestObj* pRequest = NULL;
  SQuery*      pQuery = NULL;

-  int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
+  int32_t code = buildRequest(connId, sql, sqlLen, NULL, validateOnly, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    terrno = code;
    return NULL;
  }

-  pRequest->validateOnly = validateOnly;
-
  code = parseSql(pRequest, false, &pQuery, NULL);
  if (code != TSDB_CODE_SUCCESS) {
    pRequest->code = code;
@ -944,17 +971,17 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
        SRequestConnInfo conn = {
            .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
        SSchedulerReq req = {
-          .syncReq = false,
-          .pConn = &conn,
-          .pNodeList = pNodeList,
-          .pDag = pDag,
-          .sql = pRequest->sqlstr,
-          .startTs = pRequest->metric.start,
-          .execFp = schedulerExecCb,
-          .cbParam = pRequest,
-          .chkKillFp = chkRequestKilled,
-          .chkKillParam = (void*)pRequest->self,
-          .pExecRes = NULL,
+            .syncReq = false,
+            .pConn = &conn,
+            .pNodeList = pNodeList,
+            .pDag = pDag,
+            .sql = pRequest->sqlstr,
+            .startTs = pRequest->metric.start,
+            .execFp = schedulerExecCb,
+            .cbParam = pRequest,
+            .chkKillFp = chkRequestKilled,
+            .chkKillParam = (void*)pRequest->self,
+            .pExecRes = NULL,
        };
        code = schedulerExecJob(&req, &pRequest->body.queryJob);
        taosArrayDestroy(pNodeList);
@ -973,6 +1000,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
      break;
    default:
+      pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
      break;
  }

@ -1042,19 +1070,20 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) {
  return TSDB_CODE_SUCCESS;
 }

-SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) {
+//  todo remove it soon
+SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) {
  SRequestObj* pRequest = NULL;
  int32_t      retryNum = 0;
  int32_t      code = 0;

  do {
    destroyRequest(pRequest);
-    pRequest = launchQuery(pTscObj, sql, sqlLen, validateOnly);
+    pRequest = launchQuery(connId, sql, sqlLen, validateOnly);
    if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
      break;
    }

-    code = refreshMeta(pTscObj, pRequest);
+    code = refreshMeta(pRequest->pTscObj, pRequest);
    if (code) {
      pRequest->code = code;
      break;
@ -1062,7 +1091,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool valid
  } while (retryNum++ < REQUEST_TOTAL_EXEC_TIMES);

  if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
-    removeMeta(pTscObj, pRequest->tableList);
+    removeMeta(pRequest->pTscObj, pRequest->tableList);
  }

  return pRequest;
@ -1117,7 +1146,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
    return pTscObj;
  }

-  SRequestObj* pRequest = createRequest(pTscObj, TDMT_MND_CONNECT);
+  SRequestObj* pRequest = createRequest(pTscObj->id, TDMT_MND_CONNECT);
  if (pRequest == NULL) {
    destroyTscObj(pTscObj);
    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -1395,9 +1424,9 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
    }

    SReqResultInfo* pResInfo = &pRequest->body.resInfo;
-    SSchedulerReq req = {
-      .syncReq = true,
-      .pFetchRes = (void**)&pResInfo->pData,
+    SSchedulerReq   req = {
+          .syncReq = true,
+          .pFetchRes = (void**)&pResInfo->pData,
    };
    pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
    if (pRequest->code != TSDB_CODE_SUCCESS) {
@ -1444,25 +1473,24 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
      return NULL;
    }

-    SSyncQueryParam* pParam = pRequest->body.param;
-    if (NULL == pParam) {
-      pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
-      tsem_init(&pParam->sem, 0, 0);
-    }
-
    // convert ucs4 to native multi-bytes string
    pResultInfo->convertUcs4 = convertUcs4;

+    SSyncQueryParam* pParam = pRequest->body.param;
    taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
    tsem_wait(&pParam->sem);
  }

-  if (pRequest->code == TSDB_CODE_SUCCESS && pResultInfo->numOfRows > 0 && setupOneRowPtr) {
-    doSetOneRowPtr(pResultInfo);
-    pResultInfo->current += 1;
-  }
+  if (pResultInfo->numOfRows == 0  || pRequest->code != TSDB_CODE_SUCCESS) {
+    return NULL;
+  } else {
+    if (setupOneRowPtr) {
+      doSetOneRowPtr(pResultInfo);
+      pResultInfo->current += 1;
+    }

-  return pResultInfo->row;
+    return pResultInfo->row;
+  }
 }

 static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
@ -2024,22 +2052,9 @@ void syncQueryFn(void* param, void* res, int32_t code) {
  tsem_post(&pParam->sem);
 }

-void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
-  if (NULL == taos) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    fp(param, NULL, terrno);
-    return;
-  }
-
-  int64_t  rid = *(int64_t*)taos;
-  STscObj* pTscObj = acquireTscObj(rid);
-  if (pTscObj == NULL || sql == NULL || NULL == fp) {
+void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
+  if (sql == NULL || NULL == fp) {
    terrno = TSDB_CODE_INVALID_PARA;
-    if (pTscObj) {
-      releaseTscObj(rid);
-    } else {
-      terrno = TSDB_CODE_TSC_DISCONNECTED;
-    }
    fp(param, NULL, terrno);
    return;
  }
@ -2048,26 +2063,20 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
  if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
-    releaseTscObj(rid);
-
    fp(param, NULL, terrno);
    return;
  }

  SRequestObj* pRequest = NULL;
-  int32_t      code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
+  int32_t      code = buildRequest(connId, sql, sqlLen, param, validateOnly, &pRequest);
  if (code != TSDB_CODE_SUCCESS) {
    terrno = code;
-    releaseTscObj(rid);
    fp(param, NULL, terrno);
    return;
  }

-  pRequest->validateOnly = validateOnly;
  pRequest->body.queryFp = fp;
-  pRequest->body.param = param;
  doAsyncQuery(pRequest, false);
-  releaseTscObj(rid);
 }

 TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
@ -2076,36 +2085,22 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
    return NULL;
  }

-  int64_t  rid = *(int64_t*)taos;
-  STscObj* pTscObj = acquireTscObj(rid);
-  if (pTscObj == NULL || sql == NULL) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return NULL;
-  }
-
 #if SYNC_ON_TOP_OF_ASYNC
  SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
  tsem_init(&param->sem, 0, 0);

-  taosAsyncQueryImpl((TAOS*)&rid, sql, syncQueryFn, param, validateOnly);
+  taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
  tsem_wait(&param->sem);
-
-  releaseTscObj(rid);
-
  return param->pRequest;
 #else
  size_t sqlLen = strlen(sql);
  if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
-    releaseTscObj(rid);
    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
    return NULL;
  }

-  TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
-
-  releaseTscObj(rid);
-
+  TAOS_RES* pRes = execQuery(connId, sql, sqlLen, validateOnly);
  return pRes;
 #endif
 }
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@ -479,7 +479,6 @@ void taos_stop_query(TAOS_RES *res) {
  }

  schedulerFreeJob(&pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED);
-
  tscDebug("request %" PRIx64 " killed", pRequest->requestId);
 }

@ -706,7 +705,8 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
 }

 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
-  taosAsyncQueryImpl(taos, sql, fp, param, false);
+  int64_t  connId = *(int64_t*)taos;
+  taosAsyncQueryImpl(connId, sql, fp, param, false);
 }

 int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
@ -915,7 +915,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
    return terrno;
  }

-  int64_t       rid = *(int64_t *)taos;
+  int64_t       connId = *(int64_t *)taos;
  const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024;  // 12MB list
  int32_t       code = 0;
  SRequestObj * pRequest = NULL;
@ -933,12 +933,14 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
    return TSDB_CODE_TSC_INVALID_OPERATION;
  }

-  STscObj *pTscObj = acquireTscObj(rid);
-  if (pTscObj == NULL) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    return terrno;
+  char *sql = "taos_load_table_info";
+  code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest);
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    goto _return;
  }

+  STscObj* pTscObj = pRequest->pTscObj;
  code = transferTableNameList(tableNameList, pTscObj->acctId, pTscObj->db, &catalogReq.pTableMeta);
  if (code) {
    goto _return;
@ -950,36 +952,22 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
    goto _return;
  }

-  char *sql = "taos_load_table_info";
-  code = buildRequest(pTscObj, sql, strlen(sql), &pRequest);
-  if (code != TSDB_CODE_SUCCESS) {
-    terrno = code;
-    goto _return;
-  }
-
-  SSyncQueryParam param = {0};
-  tsem_init(&param.sem, 0, 0);
-  param.pRequest = pRequest;
-
  SRequestConnInfo conn = {
      .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};

  conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);

-  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, &param, NULL);
+  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, NULL, NULL);
  if (code) {
    goto _return;
  }

-  tsem_wait(&param.sem);
+  SSyncQueryParam* pParam = pRequest->body.param;
+  tsem_wait(&pParam->sem);

 _return:
-
  taosArrayDestroy(catalogReq.pTableMeta);
  destroyRequest(pRequest);
-
-  releaseTscObj(rid);
-
  return code;
 }

--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@ -1570,41 +1570,40 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV
    return TSDB_CODE_TSC_INVALID_TIME_STAMP;
  }

+  *tsVal = timeDouble;
  size_t typeLen = strlen(type->valuestring);
  if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) {
    // seconds
-    timeDouble = timeDouble * 1e9;
+    *tsVal = *tsVal * NANOSECOND_PER_SEC;
+    timeDouble = timeDouble * NANOSECOND_PER_SEC;
    if (smlDoubleToInt64OverFlow(timeDouble)) {
      smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
      return TSDB_CODE_TSC_INVALID_TIME_STAMP;
    }
-    *tsVal = timeDouble;
  } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) {
    switch (type->valuestring[0]) {
      case 'm':
      case 'M':
        // milliseconds
-        timeDouble = timeDouble * 1e6;
+        *tsVal = *tsVal * NANOSECOND_PER_MSEC;
+        timeDouble = timeDouble * NANOSECOND_PER_MSEC;
        if (smlDoubleToInt64OverFlow(timeDouble)) {
          smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
          return TSDB_CODE_TSC_INVALID_TIME_STAMP;
        }
-        *tsVal = timeDouble;
        break;
      case 'u':
      case 'U':
        // microseconds
-        timeDouble = timeDouble * 1e3;
+        *tsVal = *tsVal * NANOSECOND_PER_USEC;
+        timeDouble = timeDouble * NANOSECOND_PER_USEC;
        if (smlDoubleToInt64OverFlow(timeDouble)) {
          smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
          return TSDB_CODE_TSC_INVALID_TIME_STAMP;
        }
-        *tsVal = timeDouble;
        break;
      case 'n':
      case 'N':
-        // nanoseconds
-        *tsVal = timeDouble;
        break;
      default:
        return TSDB_CODE_TSC_INVALID_JSON;
@ -1641,21 +1640,23 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
    if (timeDouble < 0) {
      return TSDB_CODE_TSC_INVALID_TIME_STAMP;
    }
+
    uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble);
+    tsVal = (int64_t)timeDouble;
    if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) {
-      timeDouble = timeDouble * 1e9;
+      tsVal = tsVal * NANOSECOND_PER_SEC;
+      timeDouble = timeDouble * NANOSECOND_PER_SEC;
      if (smlDoubleToInt64OverFlow(timeDouble)) {
        smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
        return TSDB_CODE_TSC_INVALID_TIME_STAMP;
      }
-      tsVal = timeDouble;
    } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-      timeDouble = timeDouble * 1e6;
+      tsVal = tsVal * NANOSECOND_PER_MSEC;
+      timeDouble = timeDouble * NANOSECOND_PER_MSEC;
      if (smlDoubleToInt64OverFlow(timeDouble)) {
        smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
        return TSDB_CODE_TSC_INVALID_TIME_STAMP;
      }
-      tsVal = timeDouble;
    } else if (timeDouble == 0) {
      tsVal = taosGetTimestampNs();
    } else {
@ -2441,22 +2442,15 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
    return NULL;
  }

-  int64_t rid = *(int64_t*)taos;
-  STscObj* pTscObj = acquireTscObj(rid);
-  if (NULL == pTscObj) {
-    terrno = TSDB_CODE_TSC_DISCONNECTED;
-    uError("SML:taos_schemaless_insert invalid taos");
-    return NULL;
-  }
-  
-  SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT);
+  SRequestObj* request = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT);
  if(!request){
-    releaseTscObj(rid);
    uError("SML:taos_schemaless_insert error request is null");
    return NULL;
  }

  int    batchs = 0;
+  STscObj* pTscObj = request->pTscObj;
+
  pTscObj->schemalessType = 1;
  SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};

@ -2506,7 +2500,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr

  batchs = ceil(((double)numLines) / LINE_BATCH);
  for (int i = 0; i < batchs; ++i) {
-    SRequestObj* req = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT);
+    SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT);
    if(!req){
      request->code = TSDB_CODE_OUT_OF_MEMORY;
      uError("SML:taos_schemaless_insert error request is null");
@ -2548,6 +2542,5 @@ end:
 //  ((STscObj *)taos)->schemalessType = 0;
  pTscObj->schemalessType = 1;
  uDebug("resultend:%s", request->msgBuf);
-  releaseTscObj(rid);
  return (TAOS_RES*)request;
 }
--- a/source/client/src/clientStmt.c
+++ b/source/client/src/clientStmt.c
@ -5,6 +5,14 @@

 #include "clientStmt.h"

+static int32_t stmtCreateRequest(STscStmt* pStmt) {
+  if (pStmt->exec.pRequest == NULL)  {
+    return buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, &pStmt->exec.pRequest);
+  } else {
+    return TSDB_CODE_SUCCESS;
+  }
+}
+
 int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
  int32_t code = 0;

@ -217,9 +225,7 @@ int32_t stmtParseSql(STscStmt* pStmt) {
      .getExecInfoFn = stmtGetExecInfo,
  };

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb));

@ -532,9 +538,7 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb,
                            pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
@ -625,9 +629,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
@ -873,9 +875,7 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
@ -905,9 +905,7 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
@ -933,10 +931,7 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
-
+  STMT_ERR_RET(stmtCreateRequest(pStmt));
  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
  }
@ -969,9 +964,7 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
    pStmt->exec.pRequest = NULL;
  }

-  if (NULL == pStmt->exec.pRequest) {
-    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
-  }
+  STMT_ERR_RET(stmtCreateRequest(pStmt));

  if (pStmt->bInfo.needParse) {
    STMT_ERR_RET(stmtParseSql(pStmt));
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
--- a/source/client/test/smlTest.cpp
+++ b/source/client/test/smlTest.cpp
@ -1284,4 +1284,210 @@ TEST(testCase, sml_dup_time_Test) {
  ASSERT_EQ(taos_errno(pRes), 0);
  taos_free_result(pRes);
 }
-*/
+
+
+TEST(testCase, sml_16960_Test) {
+TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ASSERT_NE(taos, nullptr);
+
+TAOS_RES* pRes = taos_query(taos, "create database if not exists d16368 schemaless 1");
+taos_free_result(pRes);
+
+pRes = taos_query(taos, "use d16368");
+taos_free_result(pRes);
+
+const char *sql[] = {
+    "[\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800000, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": 830525384, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800001, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -588348364, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800002, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -370310823, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800003, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -811250191, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800004, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -330340558, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "}\n"
+    "]"
+};
+
+pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+ASSERT_EQ(taos_errno(pRes), 0);
+taos_free_result(pRes);
+}
+*/
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@ -252,7 +252,7 @@ static const SSysTableMeta infosMeta[] = {
    {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
    {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
    {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
-    {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
+    // {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
    {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
    {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
    {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@ -463,6 +463,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3

  pDst->info = pBlock->info;
  pDst->info.rows = 0;
+  pDst->info.capacity = 0;
  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  for (int32_t i = 0; i < numOfCols; ++i) {
    SColumnInfoData  colInfo = {0};
@ -1356,7 +1357,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId)
  return col;
 }

-SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) {
+SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) {
  ASSERT(pBlock != NULL);
  if (index >= taosArrayGetSize(pBlock->pDataBlock)) {
    return NULL;
@ -1735,9 +1736,9 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
  int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
  int32_t rows = pDataBlock->info.rows;
  int32_t len = 0;
-  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag,
+  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "|\n", flag,
                  (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
-  if (len >= size -1) return dumpBuf;
+  if (len >= size - 1) return dumpBuf;

  for (int32_t j = 0; j < rows; j++) {
    len += snprintf(dumpBuf + len, size - len, "%s |", flag);
@ -1746,7 +1747,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
    for (int32_t k = 0; k < colNum; k++) {
      SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
      void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-      if (colDataIsNull(pColInfoData, rows, j, NULL)) {
+      if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) {
        len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
        if (len >= size -1) return dumpBuf;
        continue;
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@ -55,12 +55,11 @@ int32_t tsNumOfMnodeQueryThreads = 2;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 2;
-int32_t tsNumOfVnodeFetchThreads = 1;
+int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeWriteThreads = 2;
 int32_t tsNumOfVnodeSyncThreads = 2;
-int32_t tsNumOfVnodeMergeThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 2;
-int32_t tsNumOfQnodeFetchThreads = 1;
+int32_t tsNumOfQnodeFetchThreads = 4;
 int32_t tsNumOfSnodeSharedThreads = 2;
 int32_t tsNumOfSnodeUniqueThreads = 2;

@ -106,11 +105,6 @@ int32_t tsCompressMsgSize = -1;
 */
 int32_t tsCompressColData = -1;

-/*
- * denote if 3.0 query pattern compatible for 2.0
- */
-int32_t tsCompatibleModel = 1;
-
 // count/hyperloglog function always return values in case of all NULL data or Empty data set.
 int32_t tsCountAlwaysReturnValue = 1;

@ -190,7 +184,6 @@ int32_t tsMqRebalanceInterval = 2;
 int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushInterval = 60;

-
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
  tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
  tsDiskCfg[index].level = level;
@ -292,15 +285,14 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) {
  if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1;
  if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1;
  return 0;
 }

@ -308,7 +300,6 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
  if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0) != 0) return -1;
-  if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0) != 0) return -1;
  if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1;
@ -417,8 +408,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
  tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4);
  if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfVnodeQueryThreads = tsNumOfCores / 2;
-  tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 1);
+  tsNumOfVnodeQueryThreads = tsNumOfCores / 4;
+  tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 2);
  if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1;

  tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 1, 1);
@ -428,19 +419,16 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
  tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
  if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfVnodeSyncThreads = tsNumOfCores / 2;
+  tsNumOfVnodeSyncThreads = tsNumOfCores;
  tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1);
  if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfVnodeMergeThreads = tsNumOfCores / 8;
-  tsNumOfVnodeMergeThreads = TRANGE(tsNumOfVnodeMergeThreads, 1, 1);
-  if (cfgAddInt32(pCfg, "numOfVnodeMergeThreads", tsNumOfVnodeMergeThreads, 1, 1024, 0) != 0) return -1;
-
  tsNumOfQnodeQueryThreads = tsNumOfCores / 2;
  tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1);
  if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1;

-  tsNumOfQnodeFetchThreads = TRANGE(tsNumOfQnodeFetchThreads, 1, 1);
+  tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
+  tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
  if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;

  tsNumOfSnodeSharedThreads = tsNumOfCores / 4;
@ -470,7 +458,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {

  if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400*365, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1;
  if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 10000, 1) != 0) return -1;

  if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
@ -485,20 +473,18 @@ static void taosSetClientLogCfg(SConfig *pCfg) {
  tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
  tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
  tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
-  cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
-  uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
-  qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
-  rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
  tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
+  uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
  jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
-  idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32;
+  rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
+  qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
+  cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
 }

 static void taosSetServerLogCfg(SConfig *pCfg) {
  dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
  vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32;
  mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32;
-  qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
  wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32;
  sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32;
  tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
@ -603,7 +589,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
  tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
  tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
  tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
-  tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32;
  tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
  tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
  tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
@ -636,7 +621,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
  return 0;
 }

-int32_t taosSetCfg(SConfig *pCfg, char* name) {
+int32_t taosSetCfg(SConfig *pCfg, char *name) {
  int32_t len = strlen(name);
  char    lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
  strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len));
@ -666,7 +651,7 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
        tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
      } else if (strcasecmp("countAlwaysReturnValue", name) == 0) {
        tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
-      } else if (strcasecmp("cDebugFlag", name) == 0) {      
+      } else if (strcasecmp("cDebugFlag", name) == 0) {
        cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
      }
      break;
@ -691,10 +676,10 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
        tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN);
        tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
        snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
-        
+
        char defaultFirstEp[TSDB_EP_LEN] = {0};
        snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
-        
+
        SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp");
        SEp          firstEp = {0};
        taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
@ -704,10 +689,10 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
        tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN);
        tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
        snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
-        
+
        char defaultFirstEp[TSDB_EP_LEN] = {0};
        snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
-        
+
        SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp");
        SEp          firstEp = {0};
        taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
@ -778,7 +763,7 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
          } else if (strcasecmp("minSlidingTime", name) == 0) {
            tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32;
          } else if (strcasecmp("minIntervalTime", name) == 0) {
-            tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; 
+            tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
          } else if (strcasecmp("minimalLogDirGB", name) == 0) {
            tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval;
          }
@ -845,8 +830,6 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
        tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
      } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) {
        tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
-      } else if (strcasecmp("numOfVnodeMergeThreads", name) == 0) {
-        tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32;
      } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) {
        tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
      } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) {
@ -924,10 +907,10 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
        tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN);
        tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
        snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
-        
+
        char defaultFirstEp[TSDB_EP_LEN] = {0};
        snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
-        
+
        SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp");
        SEp          firstEp = {0};
        taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
@ -999,14 +982,13 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
      break;
    }
    default:
-      terrno = TSDB_CODE_CFG_NOT_FOUND;    
+      terrno = TSDB_CODE_CFG_NOT_FOUND;
      return -1;
  }
-  
+
  return 0;
 }

-
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
                      const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
  if (tsCfg == NULL) osDefaultInit();
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@ -381,9 +381,11 @@ static int32_t tDeserializeSClientHbRsp(SDecoder *pDecoder, SClientHbRsp *pRsp)
    if (pQnodeNum > 0) {
      pRsp->query->pQnodeList = taosArrayInit(pQnodeNum, sizeof(SQueryNodeLoad));
      if (NULL == pRsp->query->pQnodeList) return -1;
-      SQueryNodeLoad load = {0};
-      if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
-      taosArrayPush(pRsp->query->pQnodeList, &load);
+      for (int32_t i = 0; i < pQnodeNum; ++i) {
+        SQueryNodeLoad load = {0};
+        if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
+        taosArrayPush(pRsp->query->pQnodeList, &load);
+      }
    }
  }

@ -496,11 +498,18 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
  if (tStartEncode(&encoder) < 0) return -1;
  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
+  }
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
  if (tEncodeI64(&encoder, pReq->delay1) < 0) return -1;
  if (tEncodeI64(&encoder, pReq->delay2) < 0) return -1;
  if (tEncodeI64(&encoder, pReq->watermark1) < 0) return -1;
  if (tEncodeI64(&encoder, pReq->watermark2) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->colVer) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->tagVer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1;
@ -553,11 +562,18 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
  if (tStartDecode(&decoder) < 0) return -1;
  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
+  }
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
  if (tDecodeI64(&decoder, &pReq->delay1) < 0) return -1;
  if (tDecodeI64(&decoder, &pReq->delay2) < 0) return -1;
  if (tDecodeI64(&decoder, &pReq->watermark1) < 0) return -1;
  if (tDecodeI64(&decoder, &pReq->watermark2) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->colVer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->tagVer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1;
@ -645,6 +661,11 @@ int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
  if (tStartEncode(&encoder) < 0) return -1;
  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
+  }
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
  tEndEncode(&encoder);

  int32_t tlen = encoder.pos;
@ -659,6 +680,12 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq)
  if (tStartDecode(&decoder) < 0) return -1;
  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
+  }
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
+
  tEndDecode(&decoder);

  tDecoderClear(&decoder);
@ -672,8 +699,6 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
  if (tStartEncode(&encoder) < 0) return -1;
  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->tagVer) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->colVer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1;
  for (int32_t i = 0; i < pReq->numOfFields; ++i) {
    SField *pField = taosArrayGet(pReq->pFields, i);
@ -700,8 +725,6 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
  if (tStartDecode(&decoder) < 0) return -1;
  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->tagVer) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->colVer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->numOfFields) < 0) return -1;
  pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField));
  if (pReq->pFields == NULL) {
@ -1978,7 +2001,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -1991,7 +2014,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
  if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->schemaless) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1;
@ -2020,7 +2043,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -2033,7 +2056,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
  if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->schemaless) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1;
@ -2075,7 +2098,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -2083,7 +2106,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
  tEndEncode(&encoder);

@ -2101,7 +2124,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -2109,7 +2132,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
  if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
  tEndDecode(&decoder);

@ -2624,6 +2647,31 @@ int32_t tDeserializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) {
  return 0;
 }

+int32_t tSerializeSTrimDbReq(void *buf, int32_t bufLen, STrimDbReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSTrimDbReq(void *buf, int32_t bufLen, STrimDbReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
 int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
  SEncoder encoder = {0};
  tEncoderInit(&encoder, buf, bufLen);
@ -2646,7 +2694,7 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
  if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1;
  if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1;
  if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->cacheLast) < 0) return -1;
  if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1;
  for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) {
    SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i);
@ -2685,7 +2733,7 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) {
  if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1;
  if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1;
  if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->cacheLast) < 0) return -1;
  if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1;
  if (pRsp->numOfRetensions > 0) {
    pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention));
@ -3596,7 +3644,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -3611,7 +3659,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
  if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->standby) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
@ -3654,7 +3702,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -3669,7 +3717,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
  if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->standby) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
@ -3779,7 +3827,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq
  if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->pages) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->lastRowMem) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->cacheLastSize) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
  if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
@ -3787,7 +3835,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq
  if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
  for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
@ -3810,7 +3858,7 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
  if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->lastRowMem) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->cacheLastSize) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
  if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
@ -3818,7 +3866,7 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
  if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
  if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
  for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
@ -5406,11 +5454,11 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
  } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) {
    snprintf(buf, maxLen, "offset(reset to latest)");
  } else if (pVal->type == TMQ_OFFSET__LOG) {
-    snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version);
+    snprintf(buf, maxLen, "offset(log) ver:%" PRId64, pVal->version);
  } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    snprintf(buf, maxLen, "offset(ss data) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+    snprintf(buf, maxLen, "offset(ss data) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts);
  } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
-    snprintf(buf, maxLen, "offset(ss meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+    snprintf(buf, maxLen, "offset(ss meta) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts);
  } else {
    ASSERT(0);
  }
--- a/source/common/src/tname.c
+++ b/source/common/src/tname.c
@ -115,6 +115,14 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in

 #endif

+SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) {
+  pName->type = TSDB_TABLE_NAME_T;
+  pName->acctId = acctId;
+  strcpy(pName->dbname, pDbName);
+  strcpy(pName->tname, pTableName);
+  return pName;
+}
+
 int32_t tNameExtractFullName(const SName* name, char* dst) {
  assert(name != NULL && dst != NULL);

--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@ -21,7 +21,7 @@ extern SConfig *tsCfg;

 static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
  if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) {
-    dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
+    dInfo("set local info, dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
    taosThreadRwlockWrlock(&pMgmt->pData->lock);
    pMgmt->pData->dnodeId = pCfg->dnodeId;
    pMgmt->pData->clusterId = pCfg->clusterId;
--- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
@ -167,7 +167,7 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
  if (pMsg == NULL) return -1;
  memcpy(pMsg, pRpc, sizeof(SRpcMsg));

-  dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name);
+  dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
  return mmPutMsgToWorker(pMgmt, pWorker, pMsg);
 }

--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@ -210,7 +210,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
    return -1;
  }

-  dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d", createReq.vgId, createReq.isTsma, createReq.standby);
+  dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d", createReq.vgId,
+         createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize);
  vmGenerateVnodeCfg(&createReq, &vnodeCfg);

  if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) {
--- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c
@ -277,7 +277,7 @@ int32_t dmRunDnode(SDnode *pDnode) {

  while (1) {
    if (pDnode->stop) {
-      dInfo("dnode is about to stop");
+      dInfo("TDengine is about to stop");
      dmSetStatus(pDnode, DND_STAT_STOPPED);
      dmStopNodes(pDnode);
      dmCloseNodes(pDnode);
--- a/source/dnode/mgmt/test/vnode/vnode.cpp
+++ b/source/dnode/mgmt/test/vnode/vnode.cpp
@ -45,7 +45,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
    createReq.compression = 2;
    createReq.replica = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.selfIndex = 0;
    for (int r = 0; r < createReq.replica; ++r) {
      SReplica* pReplica = &createReq.replicas[r];
@ -80,7 +80,7 @@ TEST_F(DndTestVnode, 02_Alter_Vnode) {
    alterReq.walLevel = 1;
    alterReq.replica = 1;
    alterReq.strict = 1;
-    alterReq.cacheLastRow = 0;
+    alterReq.cacheLast = 0;
    alterReq.selfIndex = 0;
    for (int r = 0; r < alterReq.replica; ++r) {
      SReplica* pReplica = &alterReq.replicas[r];
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@ -246,7 +246,7 @@ typedef struct {
  int32_t buffer;
  int32_t pageSize;
  int32_t pages;
-  int32_t lastRowMem;
+  int32_t cacheLastSize;
  int32_t daysPerFile;
  int32_t daysToKeep0;
  int32_t daysToKeep1;
@ -260,7 +260,7 @@ typedef struct {
  int8_t  replications;
  int8_t  strict;
  int8_t  hashMethod;  // default is 1
-  int8_t  cacheLastRow;
+  int8_t  cacheLast;
  int32_t numOfRetensions;
  SArray* pRetensions;
  int8_t  schemaless;
--- a/source/dnode/mnode/impl/inc/mndInt.h
+++ b/source/dnode/mnode/impl/inc/mndInt.h
@ -34,8 +34,6 @@ extern "C" {
 #endif

 // clang-format off
-
-
 #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255,        __VA_ARGS__); }}
 #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255,        __VA_ARGS__); }}
 #define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("MND WARN ",  DEBUG_WARN,  255,        __VA_ARGS__); }}
@ -49,7 +47,6 @@ extern "C" {
 #define mGInfo(param, ...)  { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);}
 #define mGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);}
 #define mGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);}
-
 // clang-format on

 #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@ -15,11 +15,11 @@

 #define _DEFAULT_SOURCE
 #include "mndConsumer.h"
-#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
 #include "mndOffset.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndStb.h"
 #include "mndSubscribe.h"
@ -92,7 +92,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
  SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
  ASSERT(pConsumer);

-  mInfo("receive consumer lost msg, consumer id %ld, status %s", pLostMsg->consumerId,
+  mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId,
        mndConsumerStatusName(pConsumer->status));

  if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
@ -124,7 +124,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
  SMqConsumerObj        *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
  ASSERT(pConsumer);

-  mInfo("receive consumer recover msg, consumer id %ld, status %s", pRecoverMsg->consumerId,
+  mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId,
        mndConsumerStatusName(pConsumer->status));

  if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
@ -296,7 +296,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
  // 2. check epoch, only send ep info when epoches do not match
  if (epoch != serverEpoch) {
    taosRLockLatch(&pConsumer->lock);
-    mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
+    mInfo("process ask ep, consumer:%" PRId64 "(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
    int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics);

    rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp));
@ -435,23 +435,12 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
      goto SUBSCRIBE_OVER;
    }

-#if 0
-    // ref topic to prevent drop
-    // TODO make topic complete
-    SMqTopicObj topicObj = {0};
-    memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
-    topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1;
-    mInfo("subscribe topic %s by consumer %ld cgroup %s, refcnt %d", pTopic->name, consumerId, cgroup,
-          topicObj.refConsumerCnt);
-    if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
-#endif
-
    mndReleaseTopic(pMnode, pTopic);
  }

  pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
  if (pConsumerOld == NULL) {
-    mInfo("receive subscribe request from new consumer: %ld", consumerId);
+    mInfo("receive subscribe request from new consumer:%" PRId64, consumerId);

    pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
    tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256);
@ -472,8 +461,8 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {

    int32_t status = atomic_load_32(&pConsumerOld->status);

-    mInfo("receive subscribe request from old consumer: %ld, current status: %s", consumerId,
-          mndConsumerStatusName(status));
+    mInfo("receive subscribe request from existing consumer:%" PRId64 ", current status: %s, subscribe topic num: %d",
+          consumerId, mndConsumerStatusName(status), newTopicNum);

    if (status != MQ_CONSUMER_STATUS__READY) {
      terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
@ -849,12 +838,15 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
    pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
    if (pShow->pIter == NULL) break;
    if (taosArrayGetSize(pConsumer->assignedTopics) == 0) {
+      mDebug("showing consumer %ld no assigned topic, skip", pConsumer->consumerId);
      sdbRelease(pSdb, pConsumer);
      continue;
    }

    taosRLockLatch(&pConsumer->lock);

+    mDebug("showing consumer %ld", pConsumer->consumerId);
+
    int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics);
    bool    hasTopic = true;
    if (topicSz == 0) {
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@ -93,7 +93,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.buffer, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pageSize, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pages, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.lastRowMem, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheLastSize, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, _OVER)
@ -106,7 +106,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER)
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, _OVER)
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.strict, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLast, _OVER)
  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER)
  for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) {
@ -166,7 +166,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.buffer, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pageSize, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pages, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.lastRowMem, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheLastSize, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, _OVER)
@ -179,7 +179,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER)
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, _OVER)
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.strict, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLast, _OVER)
  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.hashMethod, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, _OVER)
  if (pDb->cfg.numOfRetensions > 0) {
@ -234,7 +234,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
  pOld->cfg.buffer = pNew->cfg.buffer;
  pOld->cfg.pageSize = pNew->cfg.pageSize;
  pOld->cfg.pages = pNew->cfg.pages;
-  pOld->cfg.lastRowMem = pNew->cfg.lastRowMem;
+  pOld->cfg.cacheLastSize = pNew->cfg.cacheLastSize;
  pOld->cfg.daysPerFile = pNew->cfg.daysPerFile;
  pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0;
  pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1;
@ -242,7 +242,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
  pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod;
  pOld->cfg.walLevel = pNew->cfg.walLevel;
  pOld->cfg.strict = pNew->cfg.strict;
-  pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow;
+  pOld->cfg.cacheLast = pNew->cfg.cacheLast;
  pOld->cfg.replications = pNew->cfg.replications;
  taosWUnLockLatch(&pOld->lock);
  return 0;
@ -291,7 +291,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
  if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
  if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1;
  if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1;
-  if (pCfg->lastRowMem < TSDB_MIN_DB_LAST_ROW_MEM || pCfg->lastRowMem > TSDB_MAX_DB_LAST_ROW_MEM) return -1;
+  if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_LAST_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_LAST_SIZE) return -1;
  if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
  if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
  if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
@ -310,7 +310,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
  if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
  if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
  if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1;
-  if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
+  if (pCfg->cacheLast < TSDB_MIN_DB_CACHE_LAST || pCfg->cacheLast > TSDB_MAX_DB_CACHE_LAST) return -1;
  if (pCfg->hashMethod != 1) return -1;
  if (pCfg->replications > mndGetDnodeSize(pMnode)) {
    terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
@ -339,8 +339,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
  if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;
  if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA;
  if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT;
-  if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
-  if (pCfg->lastRowMem <= 0) pCfg->lastRowMem = TSDB_DEFAULT_LAST_ROW_MEM;
+  if (pCfg->cacheLast < 0) pCfg->cacheLast = TSDB_DEFAULT_CACHE_LAST;
+  if (pCfg->cacheLastSize <= 0) pCfg->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE;
  if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
  if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF;
 }
@ -439,7 +439,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
      .buffer = pCreate->buffer,
      .pageSize = pCreate->pageSize,
      .pages = pCreate->pages,
-      .lastRowMem = pCreate->lastRowMem,
+      .cacheLastSize = pCreate->cacheLastSize,
      .daysPerFile = pCreate->daysPerFile,
      .daysToKeep0 = pCreate->daysToKeep0,
      .daysToKeep1 = pCreate->daysToKeep1,
@ -452,7 +452,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
      .compression = pCreate->compression,
      .replications = pCreate->replications,
      .strict = pCreate->strict,
-      .cacheLastRow = pCreate->cacheLastRow,
+      .cacheLast = pCreate->cacheLast,
      .hashMethod = 1,
      .schemaless = pCreate->schemaless,
  };
@ -623,13 +623,13 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
 #endif
  }

-  if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) {
-    pDb->cfg.cacheLastRow = pAlter->cacheLastRow;
+  if (pAlter->cacheLast >= 0 && pAlter->cacheLast != pDb->cfg.cacheLast) {
+    pDb->cfg.cacheLast = pAlter->cacheLast;
    terrno = 0;
  }

-  if (pAlter->lastRowMem > 0 && pAlter->lastRowMem != pDb->cfg.lastRowMem) {
-    pDb->cfg.lastRowMem = pAlter->lastRowMem;
+  if (pAlter->cacheLastSize > 0 && pAlter->cacheLastSize != pDb->cfg.cacheLastSize) {
+    pDb->cfg.cacheLastSize = pAlter->cacheLastSize;
    terrno = 0;
  }

@ -801,7 +801,7 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
  cfgRsp.compression = pDb->cfg.compression;
  cfgRsp.replications = pDb->cfg.replications;
  cfgRsp.strict = pDb->cfg.strict;
-  cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow;
+  cfgRsp.cacheLast = pDb->cfg.cacheLast;
  cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
  cfgRsp.pRetensions = pDb->cfg.pRetensions;
  cfgRsp.schemaless = pDb->cfg.schemaless;
@ -1467,7 +1467,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false);

    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLast, false);

    const char *precStr = NULL;
    switch (pDb->cfg.precision) {
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@ -406,7 +406,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
    }

    if (statusReq.dnodeId == 0) {
-      mInfo("dnode:%d, %s first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId);
+      mInfo("dnode:%d, %s first access, clusterId:%" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId);
    } else {
      if (statusReq.clusterId != pMnode->clusterId) {
        if (pDnode != NULL) {
--- a/source/dnode/mnode/impl/src/mndOffset.c
+++ b/source/dnode/mnode/impl/src/mndOffset.c
@ -185,7 +185,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {

  for (int32_t i = 0; i < commitOffsetReq.num; i++) {
    SMqOffset *pOffset = &commitOffsetReq.offsets[i];
-    mInfo("commit offset %ld to vg %d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId,
+    mInfo("commit offset %" PRId64 " to vgId:%d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId,
          pOffset->cgroup, pOffset->topicName);
    if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
      mError("submit offset to topic %s failed", pOffset->topicName);
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@ -527,10 +527,20 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
  streamObj.version = 1;
  streamObj.sql = pCreate->sql;
  streamObj.smaId = smaObj.uid;
-  streamObj.watermark = 0;
-  streamObj.trigger = STREAM_TRIGGER_AT_ONCE;
+  streamObj.watermark = pCreate->watermark;
+  streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE;
+  streamObj.triggerParam = pCreate->maxDelay;
  streamObj.ast = strdup(smaObj.ast);

+  // check the maxDelay
+  if (streamObj.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) {
+    int64_t msInterval = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND);
+    streamObj.triggerParam = msInterval > TSDB_MIN_ROLLUP_MAX_DELAY ? msInterval : TSDB_MIN_ROLLUP_MAX_DELAY;
+  }
+  if (streamObj.triggerParam > TSDB_MAX_ROLLUP_MAX_DELAY) {
+    streamObj.triggerParam = TSDB_MAX_ROLLUP_MAX_DELAY;
+  }
+
  if (mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg) != 0) {
    mError("sma:%s, failed to create since %s", smaObj.name, terrstr());
    return -1;
@ -1131,14 +1141,17 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
  SSmaObj *pSma = NULL;
  int32_t  cols = 0;

-  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
-  if (pDb == NULL) return 0;
+  SDbObj *pDb = NULL;
+  if (strlen(pShow->db) > 0) {
+    pDb = mndAcquireDb(pMnode, pShow->db);
+    if (pDb == NULL) return 0;
+  }

  while (numOfRows < rows) {
    pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma);
    if (pShow->pIter == NULL) break;

-    if (pSma->dbUid != pDb->uid) {
+    if (NULL != pDb && pSma->dbUid != pDb->uid) {
      sdbRelease(pSdb, pSma);
      continue;
    }
@ -1151,7 +1164,7 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
    STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName));

    char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
-    STR_TO_VARSTR(n2, (char *)mndGetDbStr(pDb->name));
+    STR_TO_VARSTR(n2, (char *)mndGetDbStr(pSma->db));

    SName stbName = {0};
    tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@ -45,6 +45,7 @@ static int32_t  mndProcessTableMetaReq(SRpcMsg *pReq);
 static int32_t  mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextStb(SMnode *pMnode, void *pIter);
 static int32_t  mndProcessTableCfgReq(SRpcMsg *pReq);
+static int32_t  mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp);

 int32_t mndInitStb(SMnode *pMnode) {
  SSdbTable table = {
@ -705,10 +706,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
  memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
  pDst->createdTime = taosGetTimestampMs();
  pDst->updateTime = pDst->createdTime;
-  pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+  pDst->uid = (pCreate->source == 1) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
  pDst->dbUid = pDb->uid;
-  pDst->tagVer = 1;
-  pDst->colVer = 1;
+  pDst->tagVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->tagVer : 1;
+  pDst->colVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->colVer : 1;
  pDst->smaVer = 1;
  pDst->nextColId = 1;
  pDst->maxdelay[0] = pCreate->delay1;
@ -854,6 +855,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
  SStbObj       *pStb = NULL;
  SDbObj        *pDb = NULL;
  SMCreateStbReq createReq = {0};
+  bool           isAlter = false;

  if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
    terrno = TSDB_CODE_INVALID_MSG;
@ -869,9 +871,39 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
  pStb = mndAcquireStb(pMnode, createReq.name);
  if (pStb != NULL) {
    if (createReq.igExists) {
-      mDebug("stb:%s, already exist, ignore exist is set", createReq.name);
-      code = 0;
-      goto _OVER;
+      if (createReq.source == TD_REQ_FROM_APP) {
+        mDebug("stb:%s, already exist, ignore exist is set", createReq.name);
+        code = 0;
+        goto _OVER;
+      } else if (pStb->uid != createReq.suid) {
+        mError("stb:%s, already exist while create, input suid:%" PRId64 " not match with exist suid:%" PRId64,
+               createReq.name, createReq.suid, pStb->uid);
+        terrno = TSDB_CODE_MND_STABLE_UID_NOT_MATCH;
+        goto _OVER;
+      } else if (createReq.tagVer > 0 || createReq.colVer > 0) {
+        int32_t tagDelta = pStb->tagVer - createReq.tagVer;
+        int32_t colDelta = pStb->colVer - createReq.colVer;
+        int32_t verDelta = tagDelta + verDelta;
+        mInfo("stb:%s, already exist while create, input tagVer:%d colVer:%d, exist tagVer:%d colVer:%d",
+              createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
+        if (tagDelta <= 0 && colDelta <= 0) {
+          mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name);
+          code = 0;
+          goto _OVER;
+        } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
+          isAlter = true;
+          mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name);
+        } else {
+          mError("stb:%s, schema version increase more than 1 number, error is returned", createReq.name);
+          terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+          goto _OVER;
+        }
+      } else {
+        mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name,
+               createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
+        terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+        goto _OVER;
+      }
    } else {
      terrno = TSDB_CODE_MND_STB_ALREADY_EXIST;
      goto _OVER;
@ -900,7 +932,12 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
    goto _OVER;
  }

-  code = mndCreateStb(pMnode, pReq, &createReq, pDb);
+  if (isAlter) {
+    bool needRsp = false;
+    code = mndAlterStbImp(pMnode, pReq, pDb, pStb, needRsp);
+  } else {
+    code = mndCreateStb(pMnode, pReq, &createReq, pDb);
+  }
  if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;

 _OVER:
@ -1466,14 +1503,13 @@ static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *t
  return code;
 }

-static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont,
-                                     int32_t *pLen) {
+static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, int32_t *pLen) {
  int32_t       ret;
  SEncoder      ec = {0};
  uint32_t      contLen = 0;
  SMAlterStbRsp alterRsp = {0};
  SName         name = {0};
-  tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+  tNameFromString(&name, pObj->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);

  alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
  if (NULL == alterRsp.pMeta) {
@ -1506,10 +1542,36 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S
  return 0;
 }

+static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp) {
+  int32_t code = -1;
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq);
+  if (pTrans == NULL) goto _OVER;
+
+  mDebug("trans:%d, used to alter stb:%s", pTrans->id, pStb->name);
+  mndTransSetDbName(pTrans, pDb->name, NULL);
+
+  if (needRsp) {
+    void   *pCont = NULL;
+    int32_t contLen = 0;
+    if (mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen) != 0) goto _OVER;
+    mndTransSetRpcRsp(pTrans, pCont, contLen);
+  }
+
+  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+
+  code = 0;
+
+_OVER:
+  mndTransDrop(pTrans);
+  return code;
+}
+
 static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) {
  bool    needRsp = true;
  int32_t code = -1;
-  STrans *pTrans = NULL;
  SField *pField0 = NULL;

  SStbObj stbObj = {0};
@ -1558,30 +1620,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
  }

  if (code != 0) goto _OVER;
-
-  code = -1;
-  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq);
-  if (pTrans == NULL) goto _OVER;
-
-  mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name);
-  mndTransSetDbName(pTrans, pDb->name, NULL);
-
-  if (needRsp) {
-    void   *pCont = NULL;
-    int32_t contLen = 0;
-    if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen) != 0) goto _OVER;
-    mndTransSetRpcRsp(pTrans, pCont, contLen);
-  }
-
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
-  if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
-  if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
-  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
-
-  code = 0;
+  code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp);

 _OVER:
-  mndTransDrop(pTrans);
  taosMemoryFreeClear(stbObj.pTags);
  taosMemoryFreeClear(stbObj.pColumns);
  return code;
@ -1614,14 +1655,6 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) {
    goto _OVER;
  }

-  if ((alterReq.tagVer > 0 && alterReq.colVer > 0) &&
-      (alterReq.tagVer <= pStb->tagVer || alterReq.colVer <= pStb->colVer)) {
-    mDebug("stb:%s, already exist, tagVer:%d colVer:%d smaller than in mnode, tagVer:%d colVer:%d, alter success",
-           alterReq.name, alterReq.tagVer, alterReq.colVer, pStb->tagVer, pStb->colVer);
-    code = 0;
-    goto _OVER;
-  }
-
  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
    goto _OVER;
  }
@ -1752,6 +1785,11 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
    }
  }

+  if (dropReq.source != TD_REQ_FROM_APP && pStb->uid != dropReq.suid) {
+    terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+    goto _OVER;
+  }
+
  pDb = mndAcquireDbByStb(pMnode, dropReq.name);
  if (pDb == NULL) {
    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@ -235,7 +235,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
            .pVgEp = pVgEp,
        };
        taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-        mInfo("mq rebalance: remove vg %d from consumer %ld", pVgEp->vgId, consumerId);
+        mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64, pVgEp->vgId, consumerId);
      }
      taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
      // put into removed
@ -255,7 +255,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
          .pVgEp = pVgEp,
      };
      taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
-      mInfo("mq rebalance: remove vg %d from unassigned", pVgEp->vgId);
+      mInfo("mq rebalance: remove vgId:%d from unassigned", pVgEp->vgId);
    }
  }

@ -298,7 +298,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
                .pVgEp = pVgEp,
            };
            taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-            mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+            mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
          }
          imbCnt++;
        }
@ -312,7 +312,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
              .pVgEp = pVgEp,
          };
          taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-          mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+          mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
        }
      }
    }
@ -329,7 +329,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
      newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
      taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
      taosArrayPush(pOutput->newConsumers, &consumerId);
-      mInfo("mq rebalance: add new consumer %ld", consumerId);
+      mInfo("mq rebalance: add new consumer:%" PRId64, consumerId);
    }
  }

@ -354,7 +354,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
      taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
      pRebVg->newConsumerId = pConsumerEp->consumerId;
      taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
    }
  }

@ -372,7 +372,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
      taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
      pRebVg->newConsumerId = pConsumerEp->consumerId;
      taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
    }
  } else {
    // if all consumer is removed, put all vg into unassigned
@ -385,7 +385,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
      ASSERT(pRebOutput->newConsumerId == -1);
      taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
      taosArrayPush(pOutput->rebVgs, pRebOutput);
-      mInfo("mq rebalance: unassign vg %d (second scan)", pRebOutput->pVgEp->vgId);
+      mInfo("mq rebalance: unassign vgId:%d (second scan)", pRebOutput->pVgEp->vgId);
    }
  }

@ -393,7 +393,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
  mInfo("rebalance calculation completed, rebalanced vg:");
  for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
    SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
-    mInfo("vgId:%d, moved from consumer %" PRId64 " to consumer %" PRId64, pOutputRebVg->pVgEp->vgId,
+    mInfo("vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId,
          pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
  }

@ -546,7 +546,11 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
      char cgroup[TSDB_CGROUP_LEN];
      mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true);
      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
-      ASSERT(pTopic);
+      /*ASSERT(pTopic);*/
+      if (pTopic == NULL) {
+        mError("rebalance %s failed since topic %s was dropped, abort", pRebInfo->key, topic);
+        continue;
+      }
      taosRLockLatch(&pTopic->lock);

      rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key);
--- a/source/dnode/mnode/impl/src/mndTopic.c
+++ b/source/dnode/mnode/impl/src/mndTopic.c
@ -558,7 +558,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
        mndReleaseConsumer(pMnode, pConsumer);
        mndReleaseTopic(pMnode, pTopic);
        terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
-        mError("topic:%s, failed to drop since subscribed by consumer %ld in consumer group %s", dropReq.name,
+        mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s", dropReq.name,
               pConsumer->consumerId, pConsumer->cgroup);
        return -1;
      }
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@ -207,7 +207,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
  createReq.buffer = pDb->cfg.buffer;
  createReq.pageSize = pDb->cfg.pageSize;
  createReq.pages = pDb->cfg.pages;
-  createReq.lastRowMem = pDb->cfg.lastRowMem;
+  createReq.cacheLastSize = pDb->cfg.cacheLastSize;
  createReq.daysPerFile = pDb->cfg.daysPerFile;
  createReq.daysToKeep0 = pDb->cfg.daysToKeep0;
  createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
@ -219,7 +219,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
  createReq.precision = pDb->cfg.precision;
  createReq.compression = pDb->cfg.compression;
  createReq.strict = pDb->cfg.strict;
-  createReq.cacheLastRow = pDb->cfg.cacheLastRow;
+  createReq.cacheLast = pDb->cfg.cacheLast;
  createReq.replica = pVgroup->replica;
  createReq.selfIndex = -1;
  createReq.hashBegin = pVgroup->hashBegin;
@ -277,7 +277,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_
  alterReq.buffer = pDb->cfg.buffer;
  alterReq.pageSize = pDb->cfg.pageSize;
  alterReq.pages = pDb->cfg.pages;
-  alterReq.lastRowMem = pDb->cfg.lastRowMem;
+  alterReq.cacheLastSize = pDb->cfg.cacheLastSize;
  alterReq.daysPerFile = pDb->cfg.daysPerFile;
  alterReq.daysToKeep0 = pDb->cfg.daysToKeep0;
  alterReq.daysToKeep1 = pDb->cfg.daysToKeep1;
@ -285,7 +285,7 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_
  alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod;
  alterReq.walLevel = pDb->cfg.walLevel;
  alterReq.strict = pDb->cfg.strict;
-  alterReq.cacheLastRow = pDb->cfg.cacheLastRow;
+  alterReq.cacheLast = pDb->cfg.cacheLast;
  alterReq.replica = pVgroup->replica;

  for (int32_t v = 0; v < pVgroup->replica; ++v) {
@ -742,8 +742,8 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDbInput, SVgObj *pVgroup) {
  int64_t vgroupMemroy = 0;
  if (pDb != NULL) {
    vgroupMemroy = (int64_t)pDb->cfg.buffer * 1024 * 1024 + (int64_t)pDb->cfg.pages * pDb->cfg.pageSize * 1024;
-    if (pDb->cfg.cacheLastRow > 0) {
-      vgroupMemroy += (int64_t)pDb->cfg.lastRowMem * 1024 * 1024;
+    if (pDb->cfg.cacheLast > 0) {
+      vgroupMemroy += (int64_t)pDb->cfg.cacheLastSize * 1024 * 1024;
    }
  }

--- a/source/dnode/mnode/impl/test/db/db.cpp
+++ b/source/dnode/mnode/impl/test/db/db.cpp
@ -50,7 +50,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;
    createReq.numOfStables = 0;
    createReq.numOfRetensions = 0;
@ -84,7 +84,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
    alterdbReq.fsyncPeriod = 4000;
    alterdbReq.walLevel = 2;
    alterdbReq.strict = 1;
-    alterdbReq.cacheLastRow = 1;
+    alterdbReq.cacheLast = 1;
    alterdbReq.replications = 1;

    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
@ -146,7 +146,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;
    createReq.numOfStables = 0;
    createReq.numOfRetensions = 0;
--- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp
+++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp
@ -288,7 +288,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;
    createReq.numOfStables = 0;
    createReq.numOfRetensions = 0;
@ -319,7 +319,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) {
    alterdbReq.fsyncPeriod = 4000;
    alterdbReq.walLevel = 2;
    alterdbReq.strict = 1;
-    alterdbReq.cacheLastRow = 1;
+    alterdbReq.cacheLast = 1;
    alterdbReq.replications = 3;

    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
@ -345,7 +345,7 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) {
    alterdbReq.fsyncPeriod = 4000;
    alterdbReq.walLevel = 2;
    alterdbReq.strict = 1;
-    alterdbReq.cacheLastRow = 1;
+    alterdbReq.cacheLast = 1;
    alterdbReq.replications = 1;

    int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq);
--- a/source/dnode/mnode/impl/test/sma/sma.cpp
+++ b/source/dnode/mnode/impl/test/sma/sma.cpp
@ -55,7 +55,7 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
  createReq.compression = 2;
  createReq.replications = 1;
  createReq.strict = 1;
-  createReq.cacheLastRow = 0;
+  createReq.cacheLast = 0;
  createReq.ignoreExist = 1;

  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/dnode/mnode/impl/test/stb/stb.cpp
+++ b/source/dnode/mnode/impl/test/stb/stb.cpp
@ -56,7 +56,7 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
  createReq.compression = 2;
  createReq.replications = 1;
  createReq.strict = 1;
-  createReq.cacheLastRow = 0;
+  createReq.cacheLast = 0;
  createReq.ignoreExist = 1;

  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
@ -277,8 +277,6 @@ void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const c
  req.numOfFields = 1;
  req.pFields = taosArrayInit(1, sizeof(SField));
  req.alterType = TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES;
-  req.tagVer = verInBlock;
-  req.colVer = verInBlock;

  SField field = {0};
  field.bytes = bytes;
@ -818,7 +816,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
  {
    void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1);
    SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
-    ASSERT_EQ(pRsp->code, 0);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);

    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
    EXPECT_EQ(test.GetShowRows(), 1);
--- a/source/dnode/mnode/impl/test/topic/topic.cpp
+++ b/source/dnode/mnode/impl/test/topic/topic.cpp
@ -48,7 +48,7 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
  createReq.compression = 2;
  createReq.replications = 1;
  createReq.strict = 1;
-  createReq.cacheLastRow = 0;
+  createReq.cacheLast = 0;
  createReq.ignoreExist = 1;

  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/dnode/mnode/impl/test/user/user.cpp
+++ b/source/dnode/mnode/impl/test/user/user.cpp
@ -315,7 +315,7 @@ TEST_F(MndTestUser, 03_Alter_User) {
    createReq.compression = 2;
    createReq.replications = 1;
    createReq.strict = 1;
-    createReq.cacheLastRow = 0;
+    createReq.cacheLast = 0;
    createReq.ignoreExist = 1;

    int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@ -111,7 +111,7 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) {

  streamSetupTrigger(pTask);

-  qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId,
+  qInfo("deploy stream: stream id %" PRId64 " task id %d child id %d on snode", pTask->streamId, pTask->taskId,
        pTask->selfChildId);

  taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *));
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@ -3,6 +3,7 @@ add_library(vnode STATIC "")
 target_sources(
    vnode
    PRIVATE
+
    # vnode
    "src/vnd/vnodeOpen.c"
    "src/vnd/vnodeBufPool.c"
@ -13,7 +14,6 @@ target_sources(
    "src/vnd/vnodeSvr.c"
    "src/vnd/vnodeSync.c"
    "src/vnd/vnodeSnapshot.c"
-    "src/vnd/vnodeUtil.c"

    # meta
    "src/meta/metaOpen.c"
@ -26,7 +26,6 @@ target_sources(
    "src/meta/metaSnapshot.c"

    # sma
-    "src/sma/sma.c"
    "src/sma/smaEnv.c"
    "src/sma/smaUtil.c"
    "src/sma/smaOpen.c"
@ -47,6 +46,7 @@ target_sources(
    "src/tsdb/tsdbUtil.c"
    "src/tsdb/tsdbSnapshot.c"
    "src/tsdb/tsdbCacheRead.c"
+    "src/tsdb/tsdbRetention.c"

    # tq
    "src/tq/tq.c"
@ -63,7 +63,6 @@ target_include_directories(
    PUBLIC "inc"
    PRIVATE "src/inc"
    PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
-    
 )
 target_link_libraries(
    vnode
@ -77,18 +76,19 @@ target_link_libraries(
    PUBLIC executor
    PUBLIC scheduler
    PUBLIC tdb
-    #PUBLIC bdb
-    #PUBLIC scalar
+
+    # PUBLIC bdb
+    # PUBLIC scalar
    PUBLIC transport
    PUBLIC stream
    PUBLIC index
 )
 target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
-if (${BUILD_WITH_INVERTEDINDEX})
-  add_definitions(-DUSE_INVERTED_INDEX) 
+
+if(${BUILD_WITH_INVERTEDINDEX})
+    add_definitions(-DUSE_INVERTED_INDEX)
 endif(${BUILD_WITH_INVERTEDINDEX})
+
 if(${BUILD_TEST})
    add_subdirectory(test)
 endif(${BUILD_TEST})
-
-
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@ -38,10 +38,11 @@ extern "C" {
 #endif

 // vnode
-typedef struct SVnode           SVnode;
-typedef struct STsdbCfg         STsdbCfg;  // todo: remove
-typedef struct SVnodeCfg        SVnodeCfg;
-typedef struct SVSnapshotReader SVSnapshotReader;
+typedef struct SVnode       SVnode;
+typedef struct STsdbCfg     STsdbCfg;  // todo: remove
+typedef struct SVnodeCfg    SVnodeCfg;
+typedef struct SVSnapReader SVSnapReader;
+typedef struct SVSnapWriter SVSnapWriter;

 extern const SVnodeCfg vnodeCfgDefault;

@ -57,10 +58,6 @@ void    vnodeStop(SVnode *pVnode);
 int64_t vnodeGetSyncHandle(SVnode *pVnode);
 void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
 void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever);
-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
-
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
 int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
@ -133,7 +130,7 @@ bool    tsdbNextDataBlock(STsdbReader *pReader);
 void    tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
 int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
 SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
-int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx);
+int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
 int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
 int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
 void   *tsdbGetIdx(SMeta *pMeta);
@ -174,6 +171,9 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
 int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
 int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);

+int32_t tqSeekVer(STqReader *pReader, int64_t ver);
+int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret);
+
 int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReader *pReader);
 bool    tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
@ -182,7 +182,14 @@ int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader);
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

-// need to reposition
+// SVSnapReader
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader);
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData);
+// SVSnapWriter
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback);
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);

 // structs
 struct STsdbCfg {
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@ -57,6 +57,9 @@ int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
 // metaCommit ==================
 static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }

+// metaTable ==================
+int metaHandleEntry(SMeta* pMeta, const SMetaEntry* pME);
+
 struct SMeta {
  TdThreadRwlock lock;

--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@ -47,7 +47,9 @@ struct SSmaEnv {
 };

 typedef struct {
-  int32_t smaRef;
+  int8_t  inited;
+  int32_t rsetId;
+  void   *tmrHandle;  // shared by all fetch tasks
 } SSmaMgmt;

 #define SMA_ENV_LOCK(env) ((env)->lock)
@ -64,7 +66,6 @@ struct SRSmaStat {
  SSma     *pSma;
  int64_t   submitVer;
  int64_t   refId;         // shared by fetch tasks
-  void     *tmrHandle;     // shared by fetch tasks
  int8_t    triggerStat;   // shared by fetch tasks
  int8_t    runningStat;   // for persistence task 
  SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
@ -81,7 +82,6 @@ struct SSmaStat {
 #define SMA_TSMA_STAT(s)     (&(s)->tsmaStat)
 #define SMA_RSMA_STAT(s)     (&(s)->rsmaStat)
 #define RSMA_INFO_HASH(r)    ((r)->rsmaInfoHash)
-#define RSMA_TMR_HANDLE(r)   ((r)->tmrHandle)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
 #define RSMA_REF_ID(r)       ((r)->refId)
@ -95,6 +95,7 @@ enum {
  TASK_TRIGGER_STAT_CANCELLED = 4,
  TASK_TRIGGER_STAT_FINISHED = 5,
 };
+
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);

@ -104,6 +105,10 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg);

 int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
 int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
+
+void   *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
+int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
+
 int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType);

 int32_t tdLockSma(SSma *pSma);
@ -183,7 +188,7 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {

 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
-void          *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat);

 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@ -89,6 +89,8 @@ typedef struct {
    STqExecTb  execTb;
    STqExecDb  execDb;
  };
+  // TODO remove it
+  int64_t tsdbEndVer;

 } STqExecHandle;

@ -129,6 +131,7 @@ typedef struct {
 static STqMgmt tqMgmt = {0};

 // tqRead
+int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset);
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);

 // tqExec
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@ -64,6 +64,7 @@ typedef struct SRowIter       SRowIter;
 typedef struct STsdbFS        STsdbFS;
 typedef struct SRowMerger     SRowMerger;
 typedef struct STsdbFSState   STsdbFSState;
+typedef struct STsdbSnapHdr   STsdbSnapHdr;

 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
@ -237,7 +238,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf);
 // tsdbCache
 int32_t tsdbOpenCache(STsdb *pTsdb);
 void    tsdbCloseCache(SLRUCache *pCache);
-int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row);
+int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
 int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup);
 int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
 int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@ -49,17 +49,20 @@
 extern "C" {
 #endif

-typedef struct SVnodeInfo          SVnodeInfo;
-typedef struct SMeta               SMeta;
-typedef struct SSma                SSma;
-typedef struct STsdb               STsdb;
-typedef struct STQ                 STQ;
-typedef struct SVState             SVState;
-typedef struct SVBufPool           SVBufPool;
-typedef struct SQWorker            SQHandle;
-typedef struct STsdbKeepCfg        STsdbKeepCfg;
-typedef struct SMetaSnapshotReader SMetaSnapshotReader;
-typedef struct STsdbSnapshotReader STsdbSnapshotReader;
+typedef struct SVnodeInfo      SVnodeInfo;
+typedef struct SMeta           SMeta;
+typedef struct SSma            SSma;
+typedef struct STsdb           STsdb;
+typedef struct STQ             STQ;
+typedef struct SVState         SVState;
+typedef struct SVBufPool       SVBufPool;
+typedef struct SQWorker        SQHandle;
+typedef struct STsdbKeepCfg    STsdbKeepCfg;
+typedef struct SMetaSnapReader SMetaSnapReader;
+typedef struct SMetaSnapWriter SMetaSnapWriter;
+typedef struct STsdbSnapReader STsdbSnapReader;
+typedef struct STsdbSnapWriter STsdbSnapWriter;
+typedef struct SSnapDataHdr    SSnapDataHdr;

 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@ -72,10 +75,8 @@ typedef struct STsdbSnapshotReader STsdbSnapshotReader;
 #define VNODE_RSMA2_DIR "rsma2"

 // vnd.h
-void*   vnodeBufPoolMalloc(SVBufPool* pPool, int size);
-void    vnodeBufPoolFree(SVBufPool* pPool, void* p);
-int32_t vnodeRealloc(void** pp, int32_t size);
-void    vnodeFree(void* p);
+void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
+void  vnodeBufPoolFree(SVBufPool* pPool, void* p);

 // meta
 typedef struct SMCtbCursor SMCtbCursor;
@ -109,9 +110,6 @@ STSma*          metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
 STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
 SArray*         metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*         metaGetSmaTbUids(SMeta* pMeta);
-int32_t         metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t         metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
-int32_t         metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
 void*           metaGetIvtIdx(SMeta* pMeta);
 int             metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
@ -131,9 +129,6 @@ int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* p
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                                void* pMemRef);
-int32_t     tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t     tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader);
-int32_t     tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData);

 // tq
 int     tqInit();
@ -163,6 +158,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
                            const char* stbFullName, int32_t vgId);

 // sma
+int32_t smaInit();
+void    smaCleanUp();
 int32_t smaOpen(SVnode* pVnode);
 int32_t smaClose(SSma* pSma);
 int32_t smaBegin(SSma* pSma);
@ -181,6 +178,23 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
 void    tdUidStoreDestory(STbUidStore* pStore);
 void*   tdUidStoreFree(STbUidStore* pStore);

+// SMetaSnapReader ========================================
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
+// SMetaSnapWriter ========================================
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback);
+// STsdbSnapReader ========================================
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader);
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData);
+// STsdbSnapWriter ========================================
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter);
+int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
+
 typedef struct {
  int8_t  streamType;  // sma or other
  int8_t  dstType;
@ -200,7 +214,9 @@ typedef struct {
 struct SVState {
  int64_t committed;
  int64_t applied;
+  int64_t applyTerm;
  int64_t commitID;
+  int64_t commitTerm;
 };

 struct SVnodeInfo {
@ -289,6 +305,12 @@ struct SSma {
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);

+struct SSnapDataHdr {
+  int8_t  type;
+  int64_t size;
+  uint8_t data[];
+};
+
 #ifdef __cplusplus
 }
 #endif
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@ -147,6 +147,8 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
      return -1;
    }

+    tDecoderClear(&pTbCur->mr.coder);
+
    metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey);
    if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) {
      continue;
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@ -15,53 +15,57 @@

 #include "meta.h"

-struct SMetaSnapshotReader {
+// SMetaSnapReader ========================================
+struct SMetaSnapReader {
  SMeta*  pMeta;
-  TBC*    pTbc;
  int64_t sver;
  int64_t ever;
+  TBC*    pTbc;
 };

-int32_t metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  int32_t              code = 0;
-  int32_t              c = 0;
-  SMetaSnapshotReader* pMetaReader = NULL;
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
+  int32_t          code = 0;
+  int32_t          c = 0;
+  SMetaSnapReader* pMetaSnapReader = NULL;

-  pMetaReader = (SMetaSnapshotReader*)taosMemoryCalloc(1, sizeof(*pMetaReader));
-  if (pMetaReader == NULL) {
+  // alloc
+  pMetaSnapReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaSnapReader));
+  if (pMetaSnapReader == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
  }
-  pMetaReader->pMeta = pMeta;
-  pMetaReader->sver = sver;
-  pMetaReader->ever = ever;
-  code = tdbTbcOpen(pMeta->pTbDb, &pMetaReader->pTbc, NULL);
+  pMetaSnapReader->pMeta = pMeta;
+  pMetaSnapReader->sver = sver;
+  pMetaSnapReader->ever = ever;
+
+  // impl
+  code = tdbTbcOpen(pMeta->pTbDb, &pMetaSnapReader->pTbc, NULL);
  if (code) {
    goto _err;
  }

-  code = tdbTbcMoveTo(pMetaReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
+  code = tdbTbcMoveTo(pMetaSnapReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
  if (code) {
    goto _err;
  }

-  *ppReader = pMetaReader;
+  *ppReader = pMetaSnapReader;
  return code;

 _err:
+  metaError("vgId:%d meta snap reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
  *ppReader = NULL;
  return code;
 }

-int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader) {
-  if (pReader) {
-    tdbTbcClose(pReader->pTbc);
-    taosMemoryFree(pReader);
-  }
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
+  tdbTbcClose((*ppReader)->pTbc);
+  taosMemoryFree(*ppReader);
+  *ppReader = NULL;
  return 0;
 }

-int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nDatap) {
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
  const void* pKey = NULL;
  const void* pData = NULL;
  int32_t     nKey = 0;
@ -71,23 +75,110 @@ int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t*
  for (;;) {
    code = tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData);
    if (code || ((STbDbKey*)pData)->version > pReader->ever) {
-      return TSDB_CODE_VND_READ_END;
+      code = TSDB_CODE_VND_READ_END;
+      goto _exit;
    }

    if (((STbDbKey*)pData)->version < pReader->sver) {
+      tdbTbcMoveToNext(pReader->pTbc);
      continue;
    }

+    tdbTbcMoveToNext(pReader->pTbc);
    break;
  }

  // copy the data
-  if (vnodeRealloc(ppData, nData) < 0) {
+  if (tRealloc(ppData, sizeof(SSnapDataHdr) + nData) < 0) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    return code;
  }
+  ((SSnapDataHdr*)(*ppData))->type = 0;  // TODO: use macro
+  ((SSnapDataHdr*)(*ppData))->size = nData;
+  memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData);

-  memcpy(*ppData, pData, nData);
-  *nDatap = nData;
+_exit:
+  return code;
+}
+
+// SMetaSnapWriter ========================================
+struct SMetaSnapWriter {
+  SMeta*  pMeta;
+  int64_t sver;
+  int64_t ever;
+};
+
+static int32_t metaSnapRollback(SMetaSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t metaSnapCommit(SMetaSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter;
+
+  // alloc
+  pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pMeta = pMeta;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  *ppWriter = pWriter;
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  *ppWriter = NULL;
+  return code;
+}
+
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter = *ppWriter;
+
+  if (rollback) {
+    code = metaSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = metaSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+  taosMemoryFree(pWriter);
+  *ppWriter = NULL;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer close failed since %s", TD_VID(pWriter->pMeta->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t    code = 0;
+  SMeta*     pMeta = pWriter->pMeta;
+  SMetaEntry metaEntry = {0};
+  SDecoder*  pDecoder = &(SDecoder){0};
+
+  tDecoderInit(pDecoder, pData, nData);
+  metaDecodeEntry(pDecoder, &metaEntry);
+
+  code = metaHandleEntry(pMeta, &metaEntry);
+  if (code) goto _err;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
  return code;
 }
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@ -17,7 +17,6 @@

 static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
 static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
@ -51,7 +50,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
  if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
    return -1;
  }
-  void *      data = pCtbEntry->ctbEntry.pTags;
+  void       *data = pCtbEntry->ctbEntry.pTags;
  const char *tagName = pSchema->name;

  tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
@ -70,7 +69,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
    char     type = pTagVal->type;

-    char *  key = pTagVal->pKey;
+    char   *key = pTagVal->pKey;
    int32_t nKey = strlen(key);

    SIndexTerm *term = NULL;
@ -78,7 +77,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
      term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
    } else if (type == TSDB_DATA_TYPE_NCHAR) {
      if (pTagVal->nData > 0) {
-        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        char   *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
        int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
        memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
        type = TSDB_DATA_TYPE_VARCHAR;
@ -109,7 +108,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
  if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
    return -1;
  }
-  void *      data = pCtbEntry->ctbEntry.pTags;
+  void       *data = pCtbEntry->ctbEntry.pTags;
  const char *tagName = pSchema->name;

  tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
@ -128,7 +127,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
    char     type = pTagVal->type;

-    char *  key = pTagVal->pKey;
+    char   *key = pTagVal->pKey;
    int32_t nKey = strlen(key);

    SIndexTerm *term = NULL;
@ -136,7 +135,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
      term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
    } else if (type == TSDB_DATA_TYPE_NCHAR) {
      if (pTagVal->nData > 0) {
-        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        char   *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
        int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
        memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
        type = TSDB_DATA_TYPE_VARCHAR;
@ -169,9 +168,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
  int         vLen = 0;
  const void *pKey = NULL;
  const void *pVal = NULL;
-  void *      pBuf = NULL;
+  void       *pBuf = NULL;
  int32_t     szBuf = 0;
-  void *      p = NULL;
+  void       *p = NULL;
  SMetaReader mr = {0};

  // validate req
@ -229,7 +228,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
  }

  // drop all child tables
-  TBC *   pCtbIdxc = NULL;
+  TBC    *pCtbIdxc = NULL;
  SArray *pArray = taosArrayInit(8, sizeof(tb_uid_t));

  tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
@ -285,8 +284,8 @@ _exit:
 int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
  SMetaEntry  oStbEntry = {0};
  SMetaEntry  nStbEntry = {0};
-  TBC *       pUidIdxc = NULL;
-  TBC *       pTbDbc = NULL;
+  TBC        *pUidIdxc = NULL;
+  TBC        *pTbDbc = NULL;
  const void *pData;
  int         nData;
  int64_t     oversion;
@ -409,7 +408,7 @@ _err:
 }

 int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
-  void *   pData = NULL;
+  void    *pData = NULL;
  int      nData = 0;
  int      rc = 0;
  tb_uid_t uid;
@ -477,7 +476,7 @@ static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
 }

 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
-  void *     pData = NULL;
+  void      *pData = NULL;
  int        nData = 0;
  int        rc = 0;
  SMetaEntry e = {0};
@ -538,14 +537,14 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
 }

 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
-  void *          pVal = NULL;
+  void           *pVal = NULL;
  int             nVal = 0;
-  const void *    pData = NULL;
+  const void     *pData = NULL;
  int             nData = 0;
  int             ret = 0;
  tb_uid_t        uid;
  int64_t         oversion;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
  SMetaEntry      entry = {0};
  SSchemaWrapper *pSchema;
  int             c;
@ -699,7 +698,7 @@ _err:
 static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
  SMetaEntry  ctbEntry = {0};
  SMetaEntry  stbEntry = {0};
-  void *      pVal = NULL;
+  void       *pVal = NULL;
  int         nVal = 0;
  int         ret;
  int         c;
@ -730,7 +729,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
  oversion = *(int64_t *)pData;

  // search table.db
-  TBC *    pTbDbc = NULL;
+  TBC     *pTbDbc = NULL;
  SDecoder dc1 = {0};
  SDecoder dc2 = {0};

@ -754,7 +753,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
  metaDecodeEntry(&dc2, &stbEntry);

  SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
  int32_t         iCol = 0;
  for (;;) {
    pColumn = NULL;
@ -784,8 +783,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
    memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
  } else {
    const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags;
-    STag *      pNewTag = NULL;
-    SArray *    pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
+    STag       *pNewTag = NULL;
+    SArray     *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
    if (!pTagArray) {
      terrno = TSDB_CODE_OUT_OF_MEMORY;
      goto _err;
@ -844,7 +843,7 @@ _err:
 }

 static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
-  void *      pVal = NULL;
+  void       *pVal = NULL;
  int         nVal = 0;
  const void *pData = NULL;
  int         nData = 0;
@ -948,8 +947,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta

 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
  STbDbKey tbDbKey;
-  void *   pKey = NULL;
-  void *   pVal = NULL;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
  int      kLen = 0;
  int      vLen = 0;
  SEncoder coder = {0};
@ -1055,14 +1054,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
 }

 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
-  void *         pData = NULL;
+  void          *pData = NULL;
  int            nData = 0;
  STbDbKey       tbDbKey = {0};
  SMetaEntry     stbEntry = {0};
-  STagIdxKey *   pTagIdxKey = NULL;
+  STagIdxKey    *pTagIdxKey = NULL;
  int32_t        nTagIdxKey;
  const SSchema *pTagColumn;       // = &stbEntry.stbEntry.schema.pSchema[0];
-  const void *   pTagData = NULL;  //
+  const void    *pTagData = NULL;  //
  int32_t        nTagData = 0;
  SDecoder       dc = {0};

@ -1109,7 +1108,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {

 static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
  SEncoder              coder = {0};
-  void *                pVal = NULL;
+  void                 *pVal = NULL;
  int                   vLen = 0;
  int                   rcode = 0;
  SSkmDbKey             skmDbKey = {0};
@ -1151,7 +1150,7 @@ _exit:
  return rcode;
 }

-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
+int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
  metaWLock(pMeta);

  // save to table.db
--- a/source/dnode/vnode/src/sma/sma.c
+++ b/source/dnode/vnode/src/sma/sma.c
@ -1,257 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "sma.h"
-
-// functions for external invocation
-
-// TODO: Who is responsible for resource allocate and release?
-int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
-  if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) {
-    smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  // TODO: destroy SSDataBlocks(msg)
-  return code;
-}
-
-int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
-  if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) {
-    smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  // TODO: destroy SSDataBlocks(msg)
-  return code;
-}
-
-int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t* days) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) {
-    smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno));
-  }
-  smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days);
-  return code;
-}
-
-
-// functions for internal invocation
-
-#if 0
-
-/**
- * @brief TODO: Assume that the final generated result it less than 3M
- *
- * @param pReq
- * @param pDataBlocks
- * @param vgId
- * @param suid  // TODO: check with Liao whether suid response is reasonable
- *
- * TODO: colId should be set
- */
-int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
-                                    tb_uid_t suid, const char* stbName, bool isCreateCtb) {
-  int32_t sz = taosArrayGetSize(pDataBlocks);
-  int32_t bufSize = sizeof(SSubmitReq);
-  for (int32_t i = 0; i < sz; ++i) {
-    SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info;
-    bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols));
-    bufSize += sizeof(SSubmitBlk);
-  }
-
-  *pReq = taosMemoryCalloc(1, bufSize);
-  if (!(*pReq)) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return TSDB_CODE_FAILED;
-  }
-  void* pDataBuf = *pReq;
-
-  SArray*     pTagArray = NULL;
-  int32_t     msgLen = sizeof(SSubmitReq);
-  int32_t     numOfBlks = 0;
-  int32_t     schemaLen = 0;
-  SRowBuilder rb = {0};
-  tdSRowInit(&rb, pTSchema->version);
-
-  for (int32_t i = 0; i < sz; ++i) {
-    SSDataBlock*    pDataBlock = taosArrayGet(pDataBlocks, i);
-    SDataBlockInfo* pDataBlkInfo = &pDataBlock->info;
-    int32_t         colNum = pDataBlkInfo->numOfCols;
-    int32_t         rows = pDataBlkInfo->rows;
-    int32_t         rowSize = pDataBlkInfo->rowSize;
-    int64_t         groupId = pDataBlkInfo->groupId;
-
-    if (rb.nCols != colNum) {
-      tdSRowSetTpInfo(&rb, colNum, pTSchema->flen);
-    }
-
-   if(isCreateCtb) {
-     SMetaReader  mr = {0};
-     const char*  ctbName = buildCtbNameByGroupId(stbName, pDataBlock->info.groupId);
-     if (metaGetTableEntryByName(&mr, ctbName) != 0) {
-       smaDebug("vgId:%d, no tsma ctb %s exists", vgId, ctbName);
-     }
-     SVCreateTbReq ctbReq = {0};
-     ctbReq.name = ctbName;
-     ctbReq.type = TSDB_CHILD_TABLE;
-     ctbReq.ctb.suid = suid;
-
-     STagVal tagVal = {.cid = colNum + PRIMARYKEY_TIMESTAMP_COL_ID,
-                       .type = TSDB_DATA_TYPE_BIGINT,
-                       .i64 = groupId};
-     STag*   pTag = NULL;
-     if(!pTagArray) {
-       pTagArray = taosArrayInit(1, sizeof(STagVal));
-       if (!pTagArray) goto _err;
-     }
-     taosArrayClear(pTagArray);
-     taosArrayPush(pTagArray, &tagVal);
-     tTagNew(pTagArray, 1, false, &pTag);
-     if (pTag == NULL) {
-       tdDestroySVCreateTbReq(&ctbReq);
-       goto _err;
-      }
-      ctbReq.ctb.pTag = (uint8_t*)pTag;
-
-      int32_t code;
-      tEncodeSize(tEncodeSVCreateTbReq, &ctbReq, schemaLen, code);
-
-      tdDestroySVCreateTbReq(&ctbReq);
-      if (code < 0) {
-        goto _err;
-      }
-   }
-
-
-
-    SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen);
-    pSubmitBlk->suid = suid;
-    pSubmitBlk->uid = groupId;
-    pSubmitBlk->numOfRows = rows;
-
-    msgLen += sizeof(SSubmitBlk);
-    int32_t dataLen = 0;
-    for (int32_t j = 0; j < rows; ++j) {                     // iterate by row
-      tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen));  // set row buf
-      bool    isStartKey = false;
-      int32_t offset = 0;
-      for (int32_t k = 0; k < colNum; ++k) {  // iterate by column
-        SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
-        STColumn*        pCol = &pTSchema->columns[k];
-        void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-        switch (pColInfoData->info.type) {
-          case TSDB_DATA_TYPE_TIMESTAMP:
-            if (!isStartKey) {
-              isStartKey = true;
-              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true,
-                                  offset, k);
-
-            } else {
-              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var,
-                                  true, offset, k);
-            }
-            break;
-          case TSDB_DATA_TYPE_NCHAR: {
-            tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true,
-                                offset, k);
-            break;
-          }
-          case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-            tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true,
-                                offset, k);
-            break;
-          }
-          case TSDB_DATA_TYPE_VARBINARY:
-          case TSDB_DATA_TYPE_DECIMAL:
-          case TSDB_DATA_TYPE_BLOB:
-          case TSDB_DATA_TYPE_JSON:
-          case TSDB_DATA_TYPE_MEDIUMBLOB:
-            uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type);
-            TASSERT(0);
-            break;
-          default:
-            if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) {
-              if (pCol->type == pColInfoData->info.type) {
-                tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset,
-                                    k);
-              } else {
-                char tv[8] = {0};
-                if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) {
-                  float v = 0;
-                  GET_TYPED_DATA(v, float, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) {
-                  double v = 0;
-                  GET_TYPED_DATA(v, double, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) {
-                  int64_t v = 0;
-                  GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                } else {
-                  uint64_t v = 0;
-                  GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                }
-                tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset,
-                                    k);
-              }
-            } else {
-              uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
-              TASSERT(0);
-            }
-            break;
-        }
-        offset += TYPE_BYTES[pCol->type];  // sum/avg would convert to int64_t/uint64_t/double during aggregation
-      }
-      dataLen += TD_ROW_LEN(rb.pBuf);
-#ifdef TD_DEBUG_PRINT_ROW
-      tdSRowPrint(rb.pBuf, pTSchema, __func__);
-#endif
-    }
-
-    ++numOfBlks;
-
-    pSubmitBlk->dataLen = dataLen;
-    msgLen += pSubmitBlk->dataLen;
-  }
-
-  (*pReq)->length = msgLen;
-
-  (*pReq)->header.vgId = htonl(vgId);
-  (*pReq)->header.contLen = htonl(msgLen);
-  (*pReq)->length = (*pReq)->header.contLen;
-  (*pReq)->numOfBlocks = htonl(numOfBlks);
-  SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1);
-  while (numOfBlks--) {
-    int32_t dataLen = blk->dataLen;
-    blk->uid = htobe64(blk->uid);
-    blk->suid = htobe64(blk->suid);
-    blk->padding = htonl(blk->padding);
-    blk->sversion = htonl(blk->sversion);
-    blk->dataLen = htonl(blk->dataLen);
-    blk->schemaLen = htonl(blk->schemaLen);
-    blk->numOfRows = htons(blk->numOfRows);
-    blk = (SSubmitBlk*)(blk->data + dataLen);
-  }
-  return TSDB_CODE_SUCCESS;
-_err:
-  taosMemoryFreeClear(*pReq);
-  taosArrayDestroy(pTagArray);
-
-  return TSDB_CODE_FAILED;
-}
-#endif
--- a/source/dnode/vnode/src/sma/smaCommit.c
+++ b/source/dnode/vnode/src/sma/smaCommit.c
@ -121,7 +121,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
  // step 3: perform persist task for qTaskInfo
  tdRSmaPersistExecImpl(pRSmaStat);

-  smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma));
+  smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma));

  return TSDB_CODE_SUCCESS;
 }
@ -173,6 +173,7 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) {
  }

  if ((pDir = taosOpenDir(dir)) == NULL) {
+    regfree(&regex);
    terrno = TAOS_SYSTEM_ERROR(errno);
    smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
    return TSDB_CODE_FAILED;
--- a/source/dnode/vnode/src/sma/smaEnv.c
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@ -18,7 +18,7 @@
 typedef struct SSmaStat SSmaStat;

 #define RSMA_TASK_INFO_HASH_SLOT 8
-#define SMA_MGMT_REF_NUM         1024
+#define SMA_MGMT_REF_NUM         10240

 extern SSmaMgmt smaMgmt;

@ -30,7 +30,73 @@ static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaE
 static void    *tdFreeTSmaStat(STSmaStat *pStat);
 static void     tdDestroyRSmaStat(void *pRSmaStat);

+/**
+ * @brief rsma init
+ *
+ * @return int32_t
+ */
 // implementation
+int32_t smaInit() {
+  int8_t  old;
+  int32_t nLoops = 0;
+  while (1) {
+    old = atomic_val_compare_exchange_8(&smaMgmt.inited, 0, 2);
+    if (old != 2) break;
+    if (++nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+
+  if (old == 0) {
+    // init tref rset
+    smaMgmt.rsetId = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
+
+    if (smaMgmt.rsetId < 0) {
+      atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma rset since %s", terrstr());
+      return TSDB_CODE_FAILED;
+    }
+
+    // init fetch timer handle
+    smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");
+    if (!smaMgmt.tmrHandle) {
+      taosCloseRef(smaMgmt.rsetId);
+      atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma tmr hanle since %s", terrstr());
+      return TSDB_CODE_FAILED;
+    }
+
+    atomic_store_8(&smaMgmt.inited, 1);
+    smaInfo("sma mgmt env is initialized, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief rsma cleanup
+ *
+ */
+void smaCleanUp() {
+  int8_t  old;
+  int32_t nLoops = 0;
+  while (1) {
+    old = atomic_val_compare_exchange_8(&smaMgmt.inited, 1, 2);
+    if (old != 2) break;
+    if (++nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+
+  if (old == 1) {
+    taosCloseRef(smaMgmt.rsetId);
+    taosTmrCleanUp(smaMgmt.tmrHandle);
+    smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
+    atomic_store_8(&smaMgmt.inited, 0);
+  }
+}

 static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) {
  SSmaEnv *pEnv = NULL;
@ -135,34 +201,24 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
      atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);

      // init smaMgmt
-      smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
-      if (smaMgmt.smaRef < 0) {
-        smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM);
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        return TSDB_CODE_FAILED;
-      }
+      smaInit();

-      int64_t refId = taosAddRef(smaMgmt.smaRef, pRSmaStat);
+      int64_t refId = taosAddRef(smaMgmt.rsetId, pRSmaStat);
      if (refId < 0) {
-        smaError("taosAddRef smaRef failed, since:%s", tstrerror(terrno));
+        smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma),
+                 refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno));
        return TSDB_CODE_FAILED;
+      } else {
+        smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId,
+                 smaMgmt.rsetId, SMA_MGMT_REF_NUM);
      }
      pRSmaStat->refId = refId;

-      // init timer
-      RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA");
-      if (!RSMA_TMR_HANDLE(pRSmaStat)) {
-        taosMemoryFreeClear(*pSmaStat);
-        return TSDB_CODE_FAILED;
-      }

      // init hash
      RSMA_INFO_HASH(pRSmaStat) = taosHashInit(
          RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
      if (!RSMA_INFO_HASH(pRSmaStat)) {
-        if (RSMA_TMR_HANDLE(pRSmaStat)) {
-          taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat));
-        }
        taosMemoryFreeClear(*pSmaStat);
        return TSDB_CODE_FAILED;
      }
@ -223,7 +279,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
      void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
      while (infoHash) {
        SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
-        tdFreeRSmaInfo(pSmaInfo);
+        tdFreeRSmaInfo(pSma, pSmaInfo);
        infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
      }
    }
@ -244,11 +300,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
        nLoops = 0;
      }
    }
-
-    // step 6: cleanup the timer handle
-    if (RSMA_TMR_HANDLE(pStat)) {
-      taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
-    }
  }
 }

@ -275,8 +326,13 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
      tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
    } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
      SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
-      if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) {
-        smaError("remove refId from rsmaRef:0x%" PRIx64 " failed since %s", RSMA_REF_ID(pRSmaStat), terrstr());
+      if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) {
+        smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", SMA_VID(pRSmaStat->pSma),
+                 RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId, terrstr());
+        ASSERT(0);
+      } else {
+        smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", SMA_VID(pRSmaStat->pSma),
+                 RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId);
      }
    } else {
      ASSERT(0);
@ -323,7 +379,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
      }
      break;
    default:
-      TASSERT(0);
+      smaError("vgId:%d undefined smaType:%", SMA_VID(pSma), smaType);
      return TSDB_CODE_FAILED;
  }

--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@ -19,23 +19,27 @@
 #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid

 SSmaMgmt smaMgmt = {
-    .smaRef = -1,
+    .inited = 0,
+    .rsetId = -1,
 };

 #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;

-static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
-static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle,
-                                       int8_t idx);
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
-                                 tb_uid_t suid, int8_t level);
-static void    tdRSmaFetchTrigger(void *param, void *tmrId);
-static void    tdRSmaPersistTrigger(void *param, void *tmrId);
-static void   *tdRSmaPersistExec(void *param);
-static void    tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
+static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static int32_t    tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                          SReadHandle *handle, int8_t idx);
+static int32_t    tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
+                                    STSchema *pTSchema, tb_uid_t suid, int8_t level);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid);
+static int32_t    tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                             int8_t blkType);
+static void       tdRSmaFetchTrigger(void *param, void *tmrId);
+static void       tdRSmaPersistTrigger(void *param, void *tmrId);
+static void      *tdRSmaPersistExec(void *param);
+static void       tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);

 static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
@ -47,25 +51,26 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);

 struct SRSmaInfoItem {
-  SRSmaInfo *pRsmaInfo;
-  int64_t    refId;
-  void      *taskInfo;  // qTaskInfo_t
-  tmr_h      tmrId;
-  int8_t     level;
-  int8_t     tmrInitFlag;
-  int8_t     triggerStat;
-  int32_t    maxDelay;
+  void   *taskInfo;  // qTaskInfo_t
+  int64_t refId;
+  tmr_h   tmrId;
+  int32_t maxDelay;
+  int8_t  level;
+  int8_t  triggerStat;
 };

 struct SRSmaInfo {
  STSchema     *pTSchema;
-  SRSmaStat    *pStat;
  int64_t       suid;
  SRSmaInfoItem items[TSDB_RETENTION_L2];
 };

-#define RSMA_INFO_SMA(r)  ((r)->pStat->pSma)
-#define RSMA_INFO_STAT(r) ((r)->pStat)
+static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
+  // adapt accordingly if definition of SRSmaInfo update
+  int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
+  ASSERT(pItem->level == 1 || pItem->level == 2);
+  return (SRSmaInfo *)POINTER_SHIFT(pItem, -sizeof(SRSmaInfoItem) * (pItem->level - 1) - rsmaInfoHeadLen);
+}

 struct SRSmaQTaskInfoItem {
  int32_t len;
@ -107,9 +112,8 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId,
  }
 }

-void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
+void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
  if (pInfo) {
-    SSma *pSma = RSMA_INFO_SMA(pInfo);
    for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
      SRSmaInfoItem *pItem = &pInfo->items[i];
      if (pItem->taskInfo) {
@ -142,8 +146,6 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
 }

 static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
-  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
  SRSmaInfo *pRSmaInfo = NULL;

  if (!suid || !tbUids) {
@ -152,8 +154,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids)
    return TSDB_CODE_FAILED;
  }

-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, *suid);
+
+  if (!pRSmaInfo) {
    smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
    terrno = TSDB_CODE_RSMA_INVALID_STAT;
    return TSDB_CODE_FAILED;
@ -251,15 +254,14 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
  return TSDB_CODE_SUCCESS;
 }

-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *pReadHandle,
-                                       int8_t idx) {
+static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                       SReadHandle *pReadHandle, int8_t idx) {
  SRetention *pRetention = SMA_RETENTION(pSma);
  STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);

  if (param->qmsg[idx]) {
    SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
-    pItem->refId = RSMA_REF_ID(pRSmaInfo->pStat);
-    pItem->pRsmaInfo = pRSmaInfo;
+    pItem->refId = RSMA_REF_ID(pStat);
    pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
    if (!pItem->taskInfo) {
      terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
@ -347,14 +349,13 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
    goto _err;
  }
  pRSmaInfo->pTSchema = pTSchema;
-  pRSmaInfo->pStat = pStat;
  pRSmaInfo->suid = suid;

-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 0) < 0) {
    goto _err;
  }

-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 1) < 0) {
    goto _err;
  }

@ -366,7 +367,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con

  return TSDB_CODE_SUCCESS;
 _err:
-  tdFreeRSmaInfo(pRSmaInfo);
+  tdFreeRSmaInfo(pSma, pRSmaInfo);
  taosMemoryFree(pReader);
  return TSDB_CODE_FAILED;
 }
@ -537,10 +538,10 @@ int64_t tdRSmaGetMaxSubmitVer(SSma *pSma, int8_t level) {
  return atomic_load_64(&pRSmaStat->submitVer);
 }

-static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
-  SArray    *pResult = NULL;
-  SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
-  SSma      *pSma = RSMA_INFO_SMA(pRSmaInfo);
+static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                          int8_t blkType) {
+  SArray *pResult = NULL;
+  SSma   *pSma = pStat->pSma;

  while (1) {
    SSDataBlock *output = NULL;
@ -572,16 +573,16 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
    STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
    SSubmitReq *pReq = NULL;
    // TODO: the schema update should be handled
-    if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) {
+    if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) {
      smaError("vgId:%d, build submit req for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
      goto _err;
    }

-    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) {
+    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pStat->submitVer, 1), pReq) < 0) {
      taosMemoryFreeClear(pReq);
      smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
      goto _err;
    }

@ -599,81 +600,16 @@ _err:
  return TSDB_CODE_FAILED;
 }

-/**
- * @brief trigger to get rsma result
- *
- * @param param
- * @param tmrId
- */
-static void tdRSmaFetchTrigger(void *param, void *tmrId) {
-  SRSmaInfoItem *pItem = param;
-  SSma          *pSma = NULL;
-  SRSmaStat     *pStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, pItem->refId);
-  if (!pStat) {
-    smaDebug("rsma fetch task not start since already destroyed");
-    return;
-  }
-
-  pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
-
-  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
-  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
-  switch (rsmaTriggerStat) {
-    case TASK_TRIGGER_STAT_PAUSED:
-    case TASK_TRIGGER_STAT_CANCELLED:
-    case TASK_TRIGGER_STAT_FINISHED: {
-      taosReleaseRef(smaMgmt.smaRef, pItem->refId);
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-      return;
-    }
-    default:
-      break;
-  }
-
-  int8_t fetchTriggerStat =
-      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
-  switch (fetchTriggerStat) {
-    case TASK_TRIGGER_STAT_ACTIVE: {
-      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-
-      tdRefSmaStat(pSma, (SSmaStat *)pStat);
-
-      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
-      tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
-
-      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
-    } break;
-    case TASK_TRIGGER_STAT_PAUSED: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INACTIVE: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INIT: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    default: {
-      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
-              SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-  }
-
-_end:
-  taosReleaseRef(smaMgmt.smaRef, pItem->refId);
-}
-
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
-                                 int8_t level) {
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
+                                 STSchema *pTSchema, tb_uid_t suid, int8_t level) {
  if (!pItem || !pItem->taskInfo) {
    smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
    return TSDB_CODE_SUCCESS;
  }
+  if (!pTSchema) {
+    smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+    return TSDB_CODE_FAILED;
+  }

  smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
           pItem->taskInfo, suid);
@ -683,14 +619,14 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
    return TSDB_CODE_FAILED;
  }

-  tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT);
-  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
-
  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
  SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);

-  if (pStat->tmrHandle) {
-    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId);
+  tdRSmaFetchAndSubmitResult(pItem, pTSchema, suid, pStat, STREAM_INPUT__DATA_SUBMIT);
+  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+
+  if (smaMgmt.tmrHandle) {
+    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
  } else {
    ASSERT(0);
  }
@ -698,19 +634,29 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
  return TSDB_CODE_SUCCESS;
 }

-static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
-  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = NULL;
  if (!pEnv) {
    // only applicable when rsma env exists
-    return TSDB_CODE_SUCCESS;
+    return NULL;
  }

-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
-  SRSmaInfo *pRSmaInfo = NULL;
-
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  if (!pStat || !RSMA_INFO_HASH(pStat)) {
+    return NULL;
+  }

+  SRSmaInfo *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+    return NULL;
+  }
+  return pRSmaInfo;
+}
+
+static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
+  if (!pRSmaInfo) {
    smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
    return TSDB_CODE_SUCCESS;
  }
@ -721,8 +667,8 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
  }

  if (inputType == STREAM_INPUT__DATA_SUBMIT) {
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1);
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
  }

  return TSDB_CODE_SUCCESS;
@ -935,13 +881,11 @@ _err:
 }

 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) {
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
  SRSmaInfo *pRSmaInfo = NULL;
  void      *qTaskInfo = NULL;

-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pItem->suid, sizeof(pItem->suid));
-
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pItem->suid);
+  if (!pRSmaInfo) {
    smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid);
    return TSDB_CODE_SUCCESS;
  }
@ -1258,7 +1202,8 @@ _end:
  }

  atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
-  taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
+  smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), smaMgmt.rsetId, pRSmaStat->refId);
+  tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__);
  taosThreadExit(NULL);
  return NULL;
 }
@ -1283,7 +1228,9 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
              atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
    }
    atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
-    taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
+    smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d)", SMA_VID(pRSmaStat->pSma), smaMgmt.rsetId,
+             pRSmaStat->refId);
+    tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__);
  }

  taosThreadAttrDestroy(&thAttr);
@ -1297,8 +1244,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
 */
 static void tdRSmaPersistTrigger(void *param, void *tmrId) {
  SRSmaStat *rsmaStat = param;
-  SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, rsmaStat->refId);
-
+  SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.rsetId, rsmaStat->refId);
+  ASSERT(0);
  if (!pRSmaStat) {
    smaDebug("rsma persistence task not start since already destroyed");
    return;
@ -1341,5 +1288,81 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
      smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
    } break;
  }
-  taosReleaseRef(smaMgmt.smaRef, rsmaStat->refId);
+  taosReleaseRef(smaMgmt.rsetId, rsmaStat->refId);
+}
+
+/**
+ * @brief trigger to get rsma result
+ *
+ * @param param
+ * @param tmrId
+ */
+static void tdRSmaFetchTrigger(void *param, void *tmrId) {
+  SRSmaInfoItem *pItem = param;
+  SSma          *pSma = NULL;
+  SRSmaStat     *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+
+  if (!pStat) {
+    smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId,
+             pItem->refId);
+    return;
+  }
+
+  pSma = pStat->pSma;
+
+  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
+  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED:
+    case TASK_TRIGGER_STAT_CANCELLED:
+    case TASK_TRIGGER_STAT_FINISHED: {
+      tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64
+               " refId:%d",
+               SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId);
+      return;
+    }
+    default:
+      break;
+  }
+
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
+
+  ASSERT(pRSmaInfo->suid > 0);
+
+  int8_t fetchTriggerStat =
+      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
+  switch (fetchTriggerStat) {
+    case TASK_TRIGGER_STAT_ACTIVE: {
+      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+
+      tdRefSmaStat(pSma, (SSmaStat *)pStat);
+
+      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
+      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
+      tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
+
+      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+    } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INACTIVE: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INIT: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+    } break;
+    default: {
+      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
+              SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+  }
+
+_end:
+  tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
 }
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@ -20,6 +20,36 @@
 #define SMA_STORAGE_MINUTES_DAY  1440
 #define SMA_STORAGE_SPLIT_FACTOR 14400  // least records in tsma file

+// TODO: Who is responsible for resource allocate and release?
+int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) {
+    smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
+  }
+  // TODO: destroy SSDataBlocks(msg)
+  return code;
+}
+
+int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) {
+    smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno));
+  }
+  // TODO: destroy SSDataBlocks(msg)
+  return code;
+}
+
+int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) {
+    smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno));
+  }
+  smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days);
+  return code;
+}
+
 /**
 * @brief Judge the tsma file split days
 *
--- a/source/dnode/vnode/src/sma/smaUtil.c
+++ b/source/dnode/vnode/src/sma/smaUtil.c
@ -294,4 +294,23 @@ int32_t tdRemoveTFile(STFile *pTFile) {
 }

 // smaXXXUtil ================
+void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) {
+  void *pResult = taosAcquireRef(rsetId, refId);
+  if (!pResult) {
+    smaWarn("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr());
+  } else {
+    smaDebug("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId);
+  }
+  return pResult;
+}
+
+int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) {
+  if (taosReleaseRef(rsetId, refId) < 0) {
+    smaWarn("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr());
+    return TSDB_CODE_FAILED;
+  }
+  smaDebug("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId);
+
+  return TSDB_CODE_SUCCESS;
+}
 // ...
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@ -112,7 +112,8 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq,
  };
  tmsgSendRsp(&resp);

-  tqDebug("vg %d from consumer %ld (epoch %d) send rsp, res msg type %d, reqOffset: %ld, rspOffset: %ld",
+  tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64
+          ", rspOffset:%" PRId64,
          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset);

  return 0;
@ -162,7 +163,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
  char buf2[80];
  tFormatOffset(buf1, 80, &pRsp->reqOffset);
  tFormatOffset(buf2, 80, &pRsp->rspOffset);
-  tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s",
+  tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, reqOffset:%s, rspOffset:%s",
          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);

  return 0;
@ -179,10 +180,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
  tDecoderClear(&decoder);

  if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey,
-            TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
+    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
+            offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
  } else if (offset.val.type == TMQ_OFFSET__LOG) {
-    tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey,
+    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey,
            TD_VID(pTq->pVnode), offset.val.version);
  } else {
    ASSERT(0);
@ -244,22 +245,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
  STqOffsetVal fetchOffsetNew;

  // 1.find handle
-  char buf[80];
-  tFormatOffset(buf, 80, &reqOffset);
-  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch,
-          TD_VID(pTq->pVnode), buf);
-
  STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
  /*ASSERT(pHandle);*/
  if (pHandle == NULL) {
-    tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
-            pReq->subKey);
+    tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId,
+            TD_VID(pTq->pVnode), pReq->subKey);
    return -1;
  }

  // check rebalance
  if (pHandle->consumerId != consumerId) {
-    tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
+    tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64
+            ", in vgId:%d, subkey %s, handle consumer id %" PRId64,
            consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
    return -1;
  }
@ -270,6 +267,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
    consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
  }

+  char buf[80];
+  tFormatOffset(buf, 80, &reqOffset);
+  tqDebug("tmq poll: consumer %ld (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId,
+          pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
+
+  SMqDataRsp dataRsp = {0};
+  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+
  // 2.reset offset if needed
  if (reqOffset.type > 0) {
    fetchOffsetNew = reqOffset;
@ -279,7 +284,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
      fetchOffsetNew = pOffset->val;
      char formatBuf[80];
      tFormatOffset(formatBuf, 80, &fetchOffsetNew);
-      tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf);
+      tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, offset reset to %s", consumerId, pHandle->subKey, formatBuf);
    } else {
      if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
        if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@ -293,25 +298,50 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
          tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
        }
      } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
-        tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal));
+        tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
+        tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %ld", consumerId, pHandle->subKey,
+                dataRsp.rspOffset.version);
+        if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+          code = -1;
+        }
+        goto OVER;
      } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
-        tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId,
-                TD_VID(pTq->pVnode), pReq->subKey);
+        tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64
+                " in vg %d, subkey %s, reset none failed",
+                pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
        terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-        return -1;
+        code = -1;
+        goto OVER;
      }
    }
  }

  // 3.query
-  SMqDataRsp dataRsp = {0};
-  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+    if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+      fetchOffsetNew.version++;
+    }
+    if (tqScan(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) {
+      ASSERT(0);
+      code = -1;
+      goto OVER;
+    }
+    if (dataRsp.blockNum == 0) {
+      // TODO add to async task pool
+      /*dataRsp.rspOffset.version--;*/
+    }
+    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+      code = -1;
+    }
+    goto OVER;
+  }

-  if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+  if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) {
    int64_t     fetchVer = fetchOffsetNew.version + 1;
    SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
    if (pCkHead == NULL) {
-      return -1;
+      code = -1;
+      goto OVER;
    }

    walSetReaderCapacity(pHandle->pWalReader, 2048);
@ -319,8 +349,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
    while (1) {
      consumerEpoch = atomic_load_32(&pHandle->epoch);
      if (consumerEpoch > reqEpoch) {
-        tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
-               consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
+        tqWarn("tmq poll: consumer %ld (epoch %d), subkey %s, vg %d offset %" PRId64
+               ", found new consumer epoch %d, discard req epoch %d",
+               consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
        break;
      }

@ -337,8 +368,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {

      SWalCont* pHead = &pCkHead->head;

-      tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
-              TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
+      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId,
+              pReq->epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType);

      if (pHead->msgType == TDMT_VND_SUBMIT) {
        SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
@ -363,7 +394,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
      } else {
        ASSERT(pHandle->fetchMeta);
        ASSERT(IS_META_MSG(pHead->msgType));
-        tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
+        tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
        SMqMetaRsp metaRsp = {0};
        /*metaRsp.reqOffset = pReq->reqOffset.version;*/
        /*metaRsp.rspOffset = fetchVer;*/
@ -383,8 +414,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
    }

    taosMemoryFree(pCkHead);
+#if 0
  } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts);
+    tqInfo("retrieve using snapshot actual offset: uid %" PRId64 " ts %" PRId64, fetchOffsetNew.uid, fetchOffsetNew.ts);
    if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
      ASSERT(0);
    }
@ -393,6 +425,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
      code = -1;
    }
+#endif
  } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) {
    ASSERT(0);
  }
@ -450,6 +483,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
    /*for (int32_t i = 0; i < 5; i++) {*/
    /*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/
    /*}*/
+    int64_t ver = walGetCommittedVer(pTq->pVnode->pWal);
    if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
      pHandle->execHandle.execCol.qmsg = req.qmsg;
      req.qmsg = NULL;
@ -460,6 +494,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
            .vnode = pTq->pVnode,
            .initTableReader = true,
            .initTqReader = true,
+            .version = ver,
        };
        pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
        ASSERT(pHandle->execHandle.execCol.task[i]);
@ -468,6 +503,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
        ASSERT(scanner);
        pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner);
        ASSERT(pHandle->execHandle.pExecReader[i]);
+        pHandle->execHandle.tsdbEndVer = ver;
      }
    } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
      for (int32_t i = 0; i < 5; i++) {
@ -476,18 +512,16 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
      pHandle->execHandle.execDb.pFilterOutTbUid =
          taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
    } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
-      for (int32_t i = 0; i < 5; i++) {
-        pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
-      }
      pHandle->execHandle.execTb.suid = req.suid;
      SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
      vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList);
-      tqDebug("vg %d, tq try get suid: %ld", pTq->pVnode->config.vgId, req.suid);
+      tqDebug("vgId:%d, tq try get suid:%" PRId64, pTq->pVnode->config.vgId, req.suid);
      for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
        int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
-        tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid);
+        tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid);
      }
      for (int32_t i = 0; i < 5; i++) {
+        pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
        tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList);
      }
      taosArrayDestroy(tbUidList);
@ -572,7 +606,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {

  streamSetupTrigger(pTask);

-  tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode));
+  tqInfo("deploy stream task id %d child id %d on vgId:%d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode));

  taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*));

--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@ -46,7 +46,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI
  return 0;
 }

-static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) {
+static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
  SMetaReader mr = {0};
  metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
  if (metaGetTableEntryByUid(&mr, uid) < 0) {
@ -59,15 +59,76 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
  return 0;
 }

+int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+  qTaskInfo_t task = pExec->execCol.task[0];
+
+  if (qStreamPrepareScan(task, pOffset) < 0) {
+    ASSERT(pOffset->type == TMQ_OFFSET__LOG);
+    pRsp->rspOffset = *pOffset;
+    pRsp->rspOffset.version--;
+    return 0;
+  }
+
+  int32_t rowCnt = 0;
+  while (1) {
+    SSDataBlock* pDataBlock = NULL;
+    uint64_t     ts = 0;
+    if (qExecTask(task, &pDataBlock, &ts) < 0) {
+      ASSERT(0);
+    }
+
+    if (pDataBlock != NULL) {
+      tqAddBlockDataToRsp(pDataBlock, pRsp);
+      pRsp->blockNum++;
+      if (pRsp->withTbName) {
+        if (pOffset->type == TMQ_OFFSET__LOG) {
+          int64_t uid = pExec->pExecReader[0]->msgIter.uid;
+          tqAddTbNameToRsp(pTq, uid, pRsp);
+        } else {
+          pRsp->withTbName = 0;
+        }
+      }
+      if (pOffset->type == TMQ_OFFSET__LOG) {
+        continue;
+      } else {
+        rowCnt += pDataBlock->info.rows;
+        if (rowCnt <= 4096) continue;
+      }
+    }
+
+    if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+      tqOffsetResetToLog(pOffset, pExec->tsdbEndVer + 1);
+      qStreamPrepareScan(task, pOffset);
+      continue;
+    }
+
+    void* meta = qStreamExtractMetaMsg(task);
+    if (meta != NULL) {
+      // tq add meta to rsp
+    }
+
+    if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
+      ASSERT(0);
+    }
+
+    ASSERT(pRsp->rspOffset.type != 0);
+
+    if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
+      ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version);
+    }
+
+    break;
+  }
+
+  return 0;
+}
+
+#if 0
 int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) {
  ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
  qTaskInfo_t task = pExec->execCol.task[workerId];

-  /*if (qStreamScanSnapshot(task) < 0) {*/
-  /*ASSERT(0);*/
-  /*}*/
-
-  if (qStreamPrepareScan(task, offset.uid, offset.ts) < 0) {
+  if (qStreamPrepareTsdbScan(task, offset.uid, offset.ts) < 0) {
    ASSERT(0);
  }

@ -93,7 +154,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S
      if (qGetStreamScanStatus(task, &uid, &ts) < 0) {
        ASSERT(0);
      }
-      tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
+      tqAddTbNameToRsp(pTq, uid, pRsp);
 #endif
    }
    pRsp->blockNum++;
@ -110,30 +171,12 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S

  return 0;
 }
+#endif

 int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) {
-  if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) {
-    qTaskInfo_t task = pExec->execCol.task[workerId];
-    ASSERT(task);
-    qSetStreamInput(task, pReq, STREAM_INPUT__DATA_SUBMIT, false);
-    while (1) {
-      SSDataBlock* pDataBlock = NULL;
-      uint64_t     ts = 0;
-      if (qExecTask(task, &pDataBlock, &ts) < 0) {
-        ASSERT(0);
-      }
-      if (pDataBlock == NULL) break;
+  ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN);

-      ASSERT(pDataBlock->info.rows != 0);
-
-      tqAddBlockDataToRsp(pDataBlock, pRsp);
-      if (pRsp->withTbName) {
-        int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
-      }
-      pRsp->blockNum++;
-    }
-  } else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
+  if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
    pRsp->withSchema = 1;
    STqReader* pReader = pExec->pExecReader[workerId];
    tqReaderSetDataMsg(pReader, pReq, 0);
@ -146,7 +189,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
      tqAddBlockDataToRsp(&block, pRsp);
      if (pRsp->withTbName) {
        int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
+        tqAddTbNameToRsp(pTq, uid, pRsp);
      }
      tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
      pRsp->blockNum++;
@ -164,15 +207,17 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
      tqAddBlockDataToRsp(&block, pRsp);
      if (pRsp->withTbName) {
        int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
+        tqAddTbNameToRsp(pTq, uid, pRsp);
      }
      tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
      pRsp->blockNum++;
    }
  }
+
  if (pRsp->blockNum == 0) {
    pRsp->skipLogNum++;
    return -1;
  }
+
  return 0;
 }
--- a/source/dnode/vnode/src/tq/tqOffset.c
+++ b/source/dnode/vnode/src/tq/tqOffset.c
@ -139,7 +139,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) {
    int64_t writeLen;
    if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) {
      ASSERT(0);
-      tqError("write offset incomplete, len %d, write len %ld", bodyLen, writeLen);
+      tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen);
      taosHashCancelIterate(pStore->pHash, pIter);
      return -1;
    }
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@ -223,7 +223,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
    memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo));
    taosWUnLockLatch(&pHandle->pushHandle.lock);

-    tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+    tqDebug("vgId:%d offset %" PRId64 " from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, reqOffset:%" PRId64 ", rspOffset:%" PRId64,
            TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum,
            rsp.reqOffset, rsp.rspOffset);

--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@ -15,11 +15,6 @@

 #include "tq.h"

-int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) {
-  /*if ()*/
-  return 0;
-}
-
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
  int32_t code = 0;
  taosThreadMutexLock(&pHandle->pWalReader->mutex);
@ -27,8 +22,8 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea

  while (1) {
    if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
-      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", pHandle->consumerId,
-              pHandle->epoch, TD_VID(pTq->pVnode), offset);
+      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
+              pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset);
      *fetchOffset = offset - 1;
      code = -1;
      goto END;
@ -84,8 +79,10 @@ STqReader* tqOpenReader(SVnode* pVnode) {
    return NULL;
  }

-  // TODO open
-  /*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/
+  pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);
+  if (pReader->pWalReader == NULL) {
+    return NULL;
+  }

  pReader->pVnodeMeta = pVnode->pMeta;
  pReader->pMsg = NULL;
@ -106,13 +103,27 @@ void tqCloseReader(STqReader* pReader) {
  taosMemoryFree(pReader);
 }

+int32_t tqSeekVer(STqReader* pReader, int64_t ver) {
+  if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
+    ASSERT(pReader->pWalReader->curInvalid);
+    ASSERT(pReader->pWalReader->curVersion == ver);
+    return -1;
+  }
+  ASSERT(pReader->pWalReader->curVersion == ver);
+  return 0;
+}
+
 int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
  bool fromProcessedMsg = pReader->pMsg != NULL;

  while (1) {
    if (!fromProcessedMsg) {
      if (walNextValidMsg(pReader->pWalReader) < 0) {
+        pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curInvalid;
+        ret->offset.type = TMQ_OFFSET__LOG;
+        ret->offset.version = pReader->ver;
        ret->fetchType = FETCH_TYPE__NONE;
+        ASSERT(ret->offset.version >= 0);
        return -1;
      }
      void* body = pReader->pWalReader->pHead->head.body;
@ -127,22 +138,21 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
    }

    while (tqNextDataBlock(pReader)) {
+      // TODO mem free
      memset(&ret->data, 0, sizeof(SSDataBlock));
      int32_t code = tqRetrieveDataBlock(&ret->data, pReader);
      if (code != 0 || ret->data.info.rows == 0) {
-        if (fromProcessedMsg) {
-          ret->fetchType = FETCH_TYPE__NONE;
-          return 0;
-        } else {
-          break;
-        }
+        ASSERT(0);
+        continue;
      }
-
      ret->fetchType = FETCH_TYPE__DATA;
      return 0;
    }

    if (fromProcessedMsg) {
+      ret->offset.type = TMQ_OFFSET__LOG;
+      ret->offset.version = pReader->ver;
+      ASSERT(pReader->ver >= 0);
      ret->fetchType = FETCH_TYPE__NONE;
      return 0;
    }
@ -179,9 +189,9 @@ bool tqNextDataBlock(STqReader* pReader) {
      return true;
    }
    void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t));
-    /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
+    /*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/
    if (ret != NULL) {
-      /*tqDebug("find   uid %ld", pHandle->msgIter.uid);*/
+      /*tqDebug("find   uid %" PRId64, pHandle->msgIter.uid);*/
      return true;
    }
  }
@ -212,7 +222,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
    if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
    pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion);
    if (pReader->pSchema == NULL) {
-      tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table",
+      tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
             pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
      /*ASSERT(0);*/
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
@ -222,7 +232,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
    if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, true);
    if (pReader->pSchemaWrapper == NULL) {
-      tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
+      tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
             pReader->msgIter.uid, pReader->cachedSchemaVer);
      /*ASSERT(0);*/
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@ -173,20 +173,64 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST
  return code;
 }

-int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row) {
+typedef struct {
+  TSKEY   ts;
+  SColVal colVal;
+} SLastCol;
+
+int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb) {
  int32_t code = 0;
  STSRow *cacheRow = NULL;
  char    key[32] = {0};
  int     keyLen = 0;

-  ((void)(row));
+  // ((void)(row));

  // getTableCacheKey(uid, "l", key, &keyLen);
  getTableCacheKey(uid, 1, key, &keyLen);
  LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen);
  if (h) {
+    STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
+    TSKEY     keyTs = row->ts;
+    bool      invalidate = false;
+
+    SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h);
+    int16_t nCol = taosArrayGetSize(pLast);
+    int16_t iCol = 0;
+
+    SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
+    if (keyTs > tTsVal->ts) {
+      STColumn *pTColumn = &pTSchema->columns[0];
+      SColVal   tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs});
+
+      taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal});
+    }
+
+    for (++iCol; iCol < nCol; ++iCol) {
+      SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
+      if (keyTs >= tTsVal->ts) {
+        SColVal *tColVal = &tTsVal->colVal;
+
+        SColVal colVal = {0};
+        tTSRowGetVal(row, pTSchema, iCol, &colVal);
+        if (colVal.isNone || colVal.isNull) {
+          if (keyTs == tTsVal->ts && !tColVal->isNone && !tColVal->isNull) {
+            invalidate = true;
+
+            break;
+          }
+        } else {
+          taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = colVal});
+        }
+      }
+    }
+
+    taosMemoryFreeClear(pTSchema);
+
+    taosLRUCacheRelease(pCache, h, invalidate);
+
    // clear last cache anyway, lazy load when get last lookup
-    taosLRUCacheRelease(pCache, h, true);
+    // taosLRUCacheRelease(pCache, h, true);
  }

  return code;
@ -516,12 +560,46 @@ typedef struct SMemNextRowIter {
  SMEMNEXTROWSTATES state;
  STbData          *pMem;  // [input]
  STbDataIter       iter;  // mem buffer skip list iterator
+  // bool              iterOpened;
+  // TSDBROW          *curRow;
 } SMemNextRowIter;

 static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow) {
+  // static int32_t getNextRowFromMem(void *iter, SArray *pRowArray) {
  SMemNextRowIter *state = (SMemNextRowIter *)iter;
  int32_t          code = 0;
+  /*
+  if (!state->iterOpened) {
+    if (state->pMem != NULL) {
+      tsdbTbDataIterOpen(state->pMem, NULL, 1, &state->iter);

+      state->iterOpened = true;
+
+      TSDBROW *pMemRow = tsdbTbDataIterGet(&state->iter);
+      if (pMemRow) {
+        state->curRow = pMemRow;
+      } else {
+        return code;
+      }
+    } else {
+      return code;
+    }
+  }
+
+  taosArrayPush(pRowArray, state->curRow);
+  while (tsdbTbDataIterNext(&state->iter)) {
+    TSDBROW *row = tsdbTbDataIterGet(&state->iter);
+
+    if (TSDBROW_TS(row) < TSDBROW_TS(state->curRow)) {
+      state->curRow = row;
+      break;
+    } else {
+      taosArrayPush(pRowArray, row);
+    }
+  }
+
+  return code;
+  */
  switch (state->state) {
    case SMEMNEXTROW_ENTER: {
      if (state->pMem != NULL) {
@ -599,7 +677,7 @@ _exit:
  return code;
 }

-static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int *iSkyline) {
+static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
  bool deleted = false;
  while (*iSkyline > 0) {
    TSDBKEY *pItemBack = (TSDBKEY *)taosArrayGet(pSkyline, *iSkyline);
@ -626,9 +704,11 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int *iSkyline) {
 }

 typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow);
+// typedef int32_t (*_next_row_fn_t)(void *iter, SArray *pRowArray);
 typedef int32_t (*_next_row_clear_fn_t)(void *iter);

-typedef struct TsdbNextRowState {
+// typedef struct TsdbNextRowState {
+typedef struct {
  TSDBROW             *pRow;
  bool                 stop;
  bool                 next;
@ -637,6 +717,388 @@ typedef struct TsdbNextRowState {
  _next_row_clear_fn_t nextRowClearFn;
 } TsdbNextRowState;

+typedef struct {
+  // STsdb  *pTsdb;
+  SArray *pSkyline;
+  int64_t iSkyline;
+
+  SBlockIdx       idx;
+  SMemNextRowIter memState;
+  SMemNextRowIter imemState;
+  SFSNextRowIter  fsState;
+  TSDBROW         memRow, imemRow, fsRow;
+
+  TsdbNextRowState input[3];
+} CacheNextRowIter;
+
+static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) {
+  int code = 0;
+
+  tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
+
+  STbData *pMem = NULL;
+  if (pTsdb->mem) {
+    tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem);
+  }
+
+  STbData *pIMem = NULL;
+  if (pTsdb->imem) {
+    tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem);
+  }
+
+  pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
+
+  SDelIdx delIdx;
+
+  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
+  if (pDelFile) {
+    SDelFReader *pDelFReader;
+
+    code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
+    if (code) goto _err;
+
+    code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
+    if (code) goto _err;
+
+    code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline);
+    if (code) goto _err;
+
+    tsdbDelFReaderClose(&pDelFReader);
+  } else {
+    code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline);
+    if (code) goto _err;
+  }
+
+  pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1;
+
+  pIter->idx = (SBlockIdx){.suid = suid, .uid = uid};
+
+  pIter->fsState.state = SFSNEXTROW_FS;
+  pIter->fsState.pTsdb = pTsdb;
+  pIter->fsState.pBlockIdxExp = &pIter->idx;
+
+  pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL};
+  pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, &pIter->imemState, getNextRowFromMem, NULL};
+  pIter->input[2] =
+      (TsdbNextRowState){&pIter->fsRow, false, true, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS};
+
+  if (pMem) {
+    pIter->memState.pMem = pMem;
+    pIter->memState.state = SMEMNEXTROW_ENTER;
+    pIter->input[0].stop = false;
+    pIter->input[0].next = true;
+  }
+
+  if (pIMem) {
+    pIter->imemState.pMem = pIMem;
+    pIter->imemState.state = SMEMNEXTROW_ENTER;
+    pIter->input[1].stop = false;
+    pIter->input[1].next = true;
+  }
+
+  return code;
+_err:
+  return code;
+}
+
+static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
+  int code = 0;
+
+  for (int i = 0; i < 3; ++i) {
+    if (pIter->input[i].nextRowClearFn) {
+      pIter->input[i].nextRowClearFn(pIter->input[i].iter);
+    }
+  }
+
+  if (pIter->pSkyline) {
+    taosArrayDestroy(pIter->pSkyline);
+  }
+
+  return code;
+_err:
+  return code;
+}
+
+// iterate next row non deleted backward ts, version (from high to low)
+static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) {
+  int code = 0;
+
+  for (int i = 0; i < 3; ++i) {
+    if (pIter->input[i].next && !pIter->input[i].stop) {
+      code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow);
+      if (code) goto _err;
+
+      if (pIter->input[i].pRow == NULL) {
+        pIter->input[i].stop = true;
+        pIter->input[i].next = false;
+      }
+    }
+  }
+
+  if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop) {
+    *ppRow = NULL;
+    return code;
+  }
+
+  // select maxpoint(s) from mem, imem, fs
+  TSDBROW *max[3] = {0};
+  int      iMax[3] = {-1, -1, -1};
+  int      nMax = 0;
+  TSKEY    maxKey = TSKEY_MIN;
+
+  for (int i = 0; i < 3; ++i) {
+    if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) {
+      TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow);
+
+      // merging & deduplicating on client side
+      if (maxKey <= key.ts) {
+        if (maxKey < key.ts) {
+          nMax = 0;
+          maxKey = key.ts;
+        }
+
+        iMax[nMax] = i;
+        max[nMax++] = pIter->input[i].pRow;
+      }
+    }
+  }
+
+  // delete detection
+  TSDBROW *merge[3] = {0};
+  int      iMerge[3] = {-1, -1, -1};
+  int      nMerge = 0;
+  for (int i = 0; i < nMax; ++i) {
+    TSDBKEY maxKey = TSDBROW_KEY(max[i]);
+
+    bool deleted = tsdbKeyDeleted(&maxKey, pIter->pSkyline, &pIter->iSkyline);
+    if (!deleted) {
+      iMerge[nMerge] = iMax[i];
+      merge[nMerge++] = max[i];
+    }
+
+    pIter->input[iMax[i]].next = deleted;
+  }
+
+  if (nMerge > 0) {
+    pIter->input[iMerge[0]].next = true;
+
+    *ppRow = merge[0];
+  } else {
+    *ppRow = NULL;
+  }
+
+  return code;
+_err:
+  return code;
+}
+
+static int32_t mergeLastRow2(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRow) {
+  int32_t code = 0;
+
+  STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
+  int16_t   nCol = pTSchema->numOfCols;
+  int16_t   iCol = 0;
+  int16_t   noneCol = 0;
+  bool      setNoneCol = false;
+  SArray   *pColArray = taosArrayInit(nCol, sizeof(SColVal));
+  SColVal  *pColVal = &(SColVal){0};
+
+  // tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
+
+  TSKEY lastRowTs = TSKEY_MAX;
+
+  CacheNextRowIter iter = {0};
+  nextRowIterOpen(&iter, uid, pTsdb);
+
+  do {
+    TSDBROW *pRow = NULL;
+    nextRowIterGet(&iter, &pRow);
+
+    if (!pRow) {
+      break;
+    }
+
+    if (lastRowTs == TSKEY_MAX) {
+      lastRowTs = TSDBROW_TS(pRow);
+      STColumn *pTColumn = &pTSchema->columns[0];
+
+      *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs});
+      if (taosArrayPush(pColArray, pColVal) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+
+      for (iCol = 1; iCol < nCol; ++iCol) {
+        tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+
+        if (taosArrayPush(pColArray, pColVal) == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
+
+        if (pColVal->isNone && !setNoneCol) {
+          noneCol = iCol;
+          setNoneCol = true;
+        }
+      }
+      if (!setNoneCol) {
+        // goto build the result ts row
+        break;
+      } else {
+        continue;
+      }
+    }
+
+    if ((TSDBROW_TS(pRow) < lastRowTs)) {
+      // goto build the result ts row
+      break;
+    }
+
+    // merge into pColArray
+    setNoneCol = false;
+    for (iCol = noneCol; iCol < nCol; ++iCol) {
+      // high version's column value
+      SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol);
+
+      tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+      if (tColVal->isNone && !pColVal->isNone) {
+        taosArraySet(pColArray, iCol, pColVal);
+      } else if (tColVal->isNone && pColVal->isNone && !setNoneCol) {
+        noneCol = iCol;
+        setNoneCol = true;
+      }
+    }
+  } while (setNoneCol);
+
+  // build the result ts row here
+  *dup = false;
+  if (taosArrayGetSize(pColArray) == nCol) {
+    code = tdSTSRowNew(pColArray, pTSchema, ppRow);
+    if (code) goto _err;
+  } else {
+    *ppRow = NULL;
+  }
+
+  nextRowIterClose(&iter);
+  taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+
+_err:
+  nextRowIterClose(&iter);
+  taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+}
+
+static int32_t mergeLast2(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
+  int32_t code = 0;
+
+  STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
+  int16_t   nCol = pTSchema->numOfCols;
+  int16_t   iCol = 0;
+  int16_t   noneCol = 0;
+  bool      setNoneCol = false;
+  SArray   *pColArray = taosArrayInit(nCol, sizeof(SLastCol));
+  SColVal  *pColVal = &(SColVal){0};
+
+  // tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
+
+  TSKEY lastRowTs = TSKEY_MAX;
+
+  CacheNextRowIter iter = {0};
+  nextRowIterOpen(&iter, uid, pTsdb);
+
+  do {
+    TSDBROW *pRow = NULL;
+    nextRowIterGet(&iter, &pRow);
+
+    if (!pRow) {
+      break;
+    }
+
+    TSKEY rowTs = TSDBROW_TS(pRow);
+
+    if (lastRowTs == TSKEY_MAX) {
+      lastRowTs = rowTs;
+      STColumn *pTColumn = &pTSchema->columns[0];
+
+      *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs});
+      if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+
+      for (iCol = 1; iCol < nCol; ++iCol) {
+        tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+
+        if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
+
+        if ((pColVal->isNone || pColVal->isNull) && !setNoneCol) {
+          noneCol = iCol;
+          setNoneCol = true;
+        }
+      }
+      if (!setNoneCol) {
+        // goto build the result ts row
+        break;
+      } else {
+        continue;
+      }
+    }
+    /*
+    if ((TSDBROW_TS(pRow) < lastRowTs)) {
+      // goto build the result ts row
+      break;
+    }
+    */
+    // merge into pColArray
+    setNoneCol = false;
+    for (iCol = noneCol; iCol < nCol; ++iCol) {
+      // high version's column value
+      SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol);
+
+      tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
+      if ((tColVal->isNone || tColVal->isNull) && (!pColVal->isNone && !pColVal->isNull)) {
+        taosArraySet(pColArray, iCol, &(SLastCol){.ts = rowTs, .colVal = *pColVal});
+        //} else if (tColVal->isNone && pColVal->isNone && !setNoneCol) {
+      } else if ((tColVal->isNone || tColVal->isNull) && (pColVal->isNone || pColVal->isNull) && !setNoneCol) {
+        noneCol = iCol;
+        setNoneCol = true;
+      }
+    }
+  } while (setNoneCol);
+
+  // build the result ts row here
+  //*dup = false;
+  if (taosArrayGetSize(pColArray) <= 0) {
+    *ppLastArray = NULL;
+    taosArrayDestroy(pColArray);
+  } else {
+    *ppLastArray = pColArray;
+  }
+  /*  if (taosArrayGetSize(pColArray) == nCol) {
+    code = tdSTSRowNew(pColArray, pTSchema, ppRow);
+    if (code) goto _err;
+  } else {
+    *ppRow = NULL;
+    }*/
+
+  nextRowIterClose(&iter);
+  // taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+
+_err:
+  nextRowIterClose(&iter);
+  // taosArrayDestroy(pColArray);
+  taosMemoryFreeClear(pTSchema);
+  return code;
+}
+
 static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRow) {
  int32_t code = 0;
  SArray *pSkyline = NULL;
@ -682,7 +1144,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
    if (code) goto _err;
  }

-  int iSkyline = taosArrayGetSize(pSkyline) - 1;
+  int64_t iSkyline = taosArrayGetSize(pSkyline) - 1;

  SBlockIdx idx = {.suid = suid, .uid = uid};

@ -719,12 +1181,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
  do {
    for (int i = 0; i < 3; ++i) {
      if (input[i].next && !input[i].stop) {
-        code = input[i].nextRowFn(input[i].iter, &input[i].pRow);
-        if (code) goto _err;
-
        if (input[i].pRow == NULL) {
-          input[i].stop = true;
-          input[i].next = false;
+          code = input[i].nextRowFn(input[i].iter, &input[i].pRow);
+          if (code) goto _err;
+
+          if (input[i].pRow == NULL) {
+            input[i].stop = true;
+            input[i].next = false;
+          }
        }
      }
    }
@ -758,14 +1222,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo

    // delete detection
    TSDBROW *merge[3] = {0};
-    // int      iMerge[3] = {-1, -1, -1};
-    int nMerge = 0;
+    int      iMerge[3] = {-1, -1, -1};
+    int      nMerge = 0;
    for (int i = 0; i < nMax; ++i) {
      TSDBKEY maxKey = TSDBROW_KEY(max[i]);

      bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline);
      if (!deleted) {
-        // iMerge[nMerge] = i;
+        iMerge[nMerge] = i;
        merge[nMerge++] = max[i];
      }

@ -792,7 +1256,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
      }
    }

-  } while (*ppRow == NULL);
+  } while (1);

  for (int i = 0; i < 3; ++i) {
    if (input[i].nextRowClearFn) {
@ -819,11 +1283,6 @@ _err:
  return code;
 }

-typedef struct {
-  TSKEY   ts;
-  SColVal colVal;
-} SLastCol;
-
 // static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
 static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
  int32_t  code = 0;
@ -873,7 +1332,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
    if (code) goto _err;
  }

-  int iSkyline = taosArrayGetSize(pSkyline) - 1;
+  int64_t iSkyline = taosArrayGetSize(pSkyline) - 1;

  SBlockIdx idx = {.suid = suid, .uid = uid};

@ -1128,7 +1587,7 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
  } else {
    STSRow *pRow = NULL;
    bool    dup = false;  // which is always false for now
-    code = mergeLastRow(uid, pTsdb, &dup, &pRow);
+    code = mergeLastRow2(uid, pTsdb, &dup, &pRow);
    // if table's empty or error, return code of -1
    if (code < 0 || pRow == NULL) {
      if (!dup && pRow) {
@ -1195,7 +1654,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand
    // STSRow *pRow = NULL;
    // code = mergeLast(uid, pTsdb, &pRow);
    SArray *pLastArray = NULL;
-    code = mergeLast(uid, pTsdb, &pLastArray);
+    // code = mergeLast(uid, pTsdb, &pLastArray);
+    code = mergeLast2(uid, pTsdb, &pLastArray);
    // if table's empty or error, return code of -1
    // if (code < 0 || pRow == NULL) {
    if (code < 0 || pLastArray == NULL) {
@ -1256,6 +1716,8 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) {

    if (invalidate) {
      taosLRUCacheRelease(pCache, h, true);
+    } else {
+      taosLRUCacheRelease(pCache, h, false);
    }
    // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen);
  }
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@ -561,7 +561,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
    }
  }

-  tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow);
+  tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb);

  pTbData->minVersion = TMIN(pTbData->minVersion, version);
  pTbData->maxVersion = TMAX(pTbData->maxVersion, version);
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@ -59,7 +59,6 @@ typedef struct SBlockLoadSuppInfo {
  SColumnDataAgg   tsColAgg;
  SColumnDataAgg** plist;
  int16_t*         colIds;    // column ids for loading file block data
-  int32_t*         slotIds;   // colId to slotId
  char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
 } SBlockLoadSuppInfo;

@ -183,7 +182,6 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
    return NULL;
  }

-  // todo apply the lastkey of table check to avoid to load header file
  for (int32_t j = 0; j < numOfTables; ++j) {
    STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
    if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
@ -218,6 +216,30 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) {
  }
 }

+static void destroyBlockScanInfo(SHashObj* pTableMap) {
+  STableBlockScanInfo* p = NULL;
+
+  while ((p = taosHashIterate(pTableMap, p)) != NULL) {
+    p->iterInit = false;
+    p->iiter.hasVal = false;
+
+    if (p->iter.iter != NULL) {
+      tsdbTbDataIterDestroy(p->iter.iter);
+      p->iter.iter = NULL;
+    }
+
+    if (p->iiter.iter != NULL) {
+      tsdbTbDataIterDestroy(p->iiter.iter);
+      p->iiter.iter = NULL;
+    }
+
+    taosArrayDestroy(p->delSkyline);
+    p->delSkyline = NULL;
+  }
+
+  taosHashCleanup(pTableMap);
+}
+
 static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) {
  ASSERT(pWindow != NULL);
  return pWindow->skey > pWindow->ekey;
@ -265,6 +287,10 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, const STsdbFSState* pFSt
  return TSDB_CODE_SUCCESS;
 }

+static void cleanupFilesetIterator(SFilesetIter* pIter) {
+  taosArrayDestroy(pIter->pFileList);
+}
+
 static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
  bool    asc = ASCENDING_TRAVERSE(pIter->order);
  int32_t step = asc ? 1 : -1;
@ -297,6 +323,9 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {

    if ((asc && (win.ekey < pReader->window.skey)) || ((!asc) && (win.skey > pReader->window.ekey))) {
      pIter->index += step;
+      if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) {
+        return false;
+      }
      continue;
    }

@ -313,7 +342,15 @@ static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) {
  pIter->order = order;
  pIter->index = -1;
  pIter->numOfBlocks = -1;
-  pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));
+  if (pIter->blockList == NULL) {
+    pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));
+  } else {
+    taosArrayClear(pIter->blockList);
+  }
+}
+
+static void cleanupDataBlockIterator(SDataBlockIter* pIter) {
+  taosArrayDestroy(pIter->blockList);
 }

 static void initReaderStatus(SReaderStatus* pStatus) {
@ -356,14 +393,14 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
  initReaderStatus(&pReader->status);

  pReader->pTsdb =
-      getTsdbByRetentions(pVnode, pCond->twindows[0].skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
+      getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
  pReader->suid = pCond->suid;
  pReader->order = pCond->order;
  pReader->capacity = 4096;
  pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL;
  pReader->verRange = getQueryVerRange(pVnode, pCond, level);
  pReader->type = pCond->type;
-  pReader->window = updateQueryTimeWindow(pVnode->pTsdb, pCond->twindows);
+  pReader->window = updateQueryTimeWindow(pVnode->pTsdb, &pCond->twindows);

  ASSERT(pCond->numOfCols > 0);

@ -2182,12 +2219,21 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
  return VND_TSDB(pVnode);
 }

-static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) {
+SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) {
+  int64_t startVer = (pCond->startVersion == -1)? 0:pCond->startVersion;
+
  if (VND_IS_RSMA(pVnode)) {
-    return (SVersionRange){.minVer = pCond->startVersion, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)};
+    return (SVersionRange){.minVer = startVer, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)};
  }

-  return (SVersionRange){.minVer = pCond->startVersion, .maxVer = pVnode->state.applied};
+  int64_t endVer = 0;
+  if (pCond->endVersion == -1) {  // user not specified end version, set current maximum version of vnode as the endVersion
+    endVer = pVnode->state.applied;
+  } else {
+    endVer = (pCond->endVersion > pVnode->state.applied)? pVnode->state.applied:pCond->endVersion;
+  }
+
+  return (SVersionRange){.minVer = startVer, .maxVer = endVer};
 }

 // // todo not unref yet, since it is not support multi-group interpolation query
@ -2506,6 +2552,7 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe
  tRowMergerInit(&merge, pRow, pReader->pSchema);
  doMergeRowsInBuf(pIter, k.ts, pDelList, &merge, pReader);
  tRowMergerGetRow(&merge, pTSRow);
+  tRowMergerClear(&merge);
 }

 void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader,
@ -2642,6 +2689,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
    }

    doAppendOneRow(pBlock, pReader, pTSRow);
+    taosMemoryFree(pTSRow);

    // no data in buffer, return immediately
    if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) {
@ -2769,11 +2817,24 @@ void tsdbReaderClose(STsdbReader* pReader) {
    return;
  }

-  blockDataDestroy(pReader->pResBlock);
-  taosMemoryFreeClear(pReader->suppInfo.plist);
+  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
+
+  taosMemoryFreeClear(pSupInfo->plist);
+  taosMemoryFree(pSupInfo->colIds);
+
+  taosArrayDestroy(pSupInfo->pColAgg);
+  for(int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) {
+    if (pSupInfo->buildBuf[i] != NULL) {
+      taosMemoryFreeClear(pSupInfo->buildBuf[i]);
+    }
+  }
+  taosMemoryFree(pSupInfo->buildBuf);
+
+  cleanupFilesetIterator(&pReader->status.fileIter);
+  cleanupDataBlockIterator(&pReader->status.blockIter);
+  destroyBlockScanInfo(pReader->status.pTableMap);
+  blockDataDestroy(pReader->pResBlock);

-  taosArrayDestroy(pReader->suppInfo.pColAgg);
-  taosMemoryFree(pReader->suppInfo.slotIds);

 #if 0
 //   if (pReader->status.pTableScanInfo != NULL) {
@ -2945,7 +3006,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
  return pReader->pResBlock->pDataBlock;
 }

-int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) {
+int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
  if (isEmptyQueryTimeWindow(&pReader->window)) {
    return TSDB_CODE_SUCCESS;
  }
@ -2955,7 +3016,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_
  pReader->status.loadFromFile = true;
  pReader->status.pTableIter = NULL;

-  pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows[tWinIdx]);
+  pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows);

  // allocate buffer in order to load data blocks from file
  memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));
--- a/source/dnode/vnode/src/tsdb/tsdbRetention.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@ -13,33 +13,30 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

-#include "vnd.h"
+#include "tsdb.h"

-int32_t vnodeRealloc(void** pp, int32_t size) {
-  uint8_t* p = NULL;
-  int32_t  csize = 0;
+int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
+  int32_t code = 0;

-  if (*pp) {
-    p = (uint8_t*)(*pp) - sizeof(int32_t);
-    csize = *(int32_t*)p;
+  // begin
+  code = tsdbFSBegin(pTsdb->fs);
+  if (code) goto _err;
+
+  // do retention
+  for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs->nState->aDFileSet); iSet++) {
+    SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pTsdb->fs->nState->aDFileSet, iSet);
+
+    // TODO
  }

-  if (csize >= size) {
-    return 0;
-  }
+  // commit
+  code = tsdbFSCommit(pTsdb->fs);
+  if (code) goto _err;

-  p = (uint8_t*)taosMemoryRealloc(p, size);
-  if (p == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  *(int32_t*)p = size;
-  *pp = p + sizeof(int32_t);
+_exit:
+  return code;

-  return 0;
-}
-
-void vnodeFree(void* p) {
-  if (p) {
-    taosMemoryFree(((uint8_t*)p) - sizeof(int32_t));
-  }
+_err:
+  tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
 }
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@ -15,22 +15,686 @@

 #include "tsdb.h"

-struct STsdbSnapshotReader {
-  STsdb* pTsdb;
-  // TODO
+// STsdbSnapReader ========================================
+struct STsdbSnapReader {
+  STsdb*  pTsdb;
+  int64_t sver;
+  int64_t ever;
+  // for data file
+  int8_t        dataDone;
+  int32_t       fid;
+  SDataFReader* pDataFReader;
+  SArray*       aBlockIdx;  // SArray<SBlockIdx>
+  int32_t       iBlockIdx;
+  SBlockIdx*    pBlockIdx;
+  SMapData      mBlock;  // SMapData<SBlock>
+  int32_t       iBlock;
+  SBlockData    blkData;
+  // for del file
+  int8_t       delDone;
+  SDelFReader* pDelFReader;
+  int32_t      iDelIdx;
+  SArray*      aDelIdx;   // SArray<SDelIdx>
+  SArray*      aDelData;  // SArray<SDelData>
 };

-int32_t tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  // TODO
-  return 0;
+static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t code = 0;
+
+  while (true) {
+    if (pReader->pDataFReader == NULL) {
+      SDFileSet* pSet = NULL;
+
+      // search the next data file set to read (todo)
+      if (0 /* TODO */) {
+        code = TSDB_CODE_VND_READ_END;
+        goto _exit;
+      }
+
+      // open
+      code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
+      if (code) goto _err;
+
+      // SBlockIdx
+      code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL);
+      if (code) goto _err;
+
+      pReader->iBlockIdx = 0;
+      pReader->pBlockIdx = NULL;
+    }
+
+    while (true) {
+      if (pReader->pBlockIdx == NULL) {
+        if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) {
+          tsdbDataFReaderClose(&pReader->pDataFReader);
+          break;
+        }
+
+        pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+        pReader->iBlockIdx++;
+
+        // SBlock
+        code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL);
+        if (code) goto _err;
+
+        pReader->iBlock = 0;
+      }
+
+      while (true) {
+        SBlock  block;
+        SBlock* pBlock = &block;
+
+        if (pReader->iBlock >= pReader->mBlock.nItem) {
+          pReader->pBlockIdx = NULL;
+          break;
+        }
+
+        tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock);
+        pReader->iBlock++;
+
+        if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) ||
+            (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) {
+          // overlap (todo)
+
+          code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->blkData, NULL, NULL);
+          if (code) goto _err;
+
+          goto _exit;
+        }
+      }
+    }
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d snap read data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
 }

-int32_t tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader) {
-  // TODO
-  return 0;
+static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t   code = 0;
+  STsdb*    pTsdb = pReader->pTsdb;
+  SDelFile* pDelFile = pTsdb->fs->cState->pDelFile;
+
+  if (pReader->pDelFReader == NULL) {
+    if (pDelFile == NULL) {
+      code = TSDB_CODE_VND_READ_END;
+      goto _exit;
+    }
+
+    // open
+    code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb, NULL);
+    if (code) goto _err;
+
+    // read index
+    code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx, NULL);
+    if (code) goto _err;
+
+    pReader->iDelIdx = 0;
+  }
+
+  while (pReader->iDelIdx < taosArrayGetSize(pReader->aDelIdx)) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx);
+    int32_t  size = 0;
+
+    pReader->iDelIdx++;
+
+    code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL);
+    if (code) goto _err;
+
+    for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
+      SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
+
+      if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
+        size += tPutDelData(NULL, pDelData);
+      }
+    }
+
+    if (size > 0) {
+      int64_t n = 0;
+
+      size = size + sizeof(SSnapDataHdr) + sizeof(TABLEID);
+      code = tRealloc(ppData, size);
+      if (code) goto _err;
+
+      // SSnapDataHdr
+      SSnapDataHdr* pSnapDataHdr = (SSnapDataHdr*)(*ppData + n);
+      pSnapDataHdr->type = 1;
+      pSnapDataHdr->size = size;  // TODO: size here may incorrect
+      n += sizeof(SSnapDataHdr);
+
+      // TABLEID
+      TABLEID* pId = (TABLEID*)(*ppData + n);
+      pId->suid = pDelIdx->suid;
+      pId->uid = pDelIdx->uid;
+      n += sizeof(*pId);
+
+      // DATA
+      for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
+        SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
+
+        if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
+          n += tPutDelData(*ppData + n, pDelData);
+        }
+      }
+
+      goto _exit;
+    }
+  }
+
+  code = TSDB_CODE_VND_READ_END;
+  tsdbDelFReaderClose(&pReader->pDelFReader);
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d snap read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
 }

-int32_t tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData) {
-  // TODO
-  return 0;
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader) {
+  int32_t          code = 0;
+  STsdbSnapReader* pReader = NULL;
+
+  // alloc
+  pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
+  if (pReader == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pReader->pTsdb = pTsdb;
+  pReader->sver = sver;
+  pReader->ever = ever;
+
+  pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+  if (pReader->aBlockIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pReader->mBlock = tMapDataInit();
+
+  code = tBlockDataInit(&pReader->blkData);
+  if (code) goto _err;
+
+  pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
+  if (pReader->aDelIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
+  if (pReader->aDelData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  *ppReader = pReader;
+  return code;
+
+_err:
+  tsdbError("vgId:%d snapshot reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppReader = NULL;
+  return code;
+}
+
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
+  int32_t          code = 0;
+  STsdbSnapReader* pReader = *ppReader;
+
+  taosArrayDestroy(pReader->aDelData);
+  taosArrayDestroy(pReader->aDelIdx);
+  if (pReader->pDelFReader) {
+    tsdbDelFReaderClose(&pReader->pDelFReader);
+  }
+  tBlockDataClear(&pReader->blkData);
+  tMapDataClear(&pReader->mBlock);
+  taosArrayDestroy(pReader->aBlockIdx);
+  if (pReader->pDataFReader) {
+    tsdbDataFReaderClose(&pReader->pDataFReader);
+  }
+  taosMemoryFree(pReader);
+  *ppReader = NULL;
+
+  return code;
+}
+
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t code = 0;
+
+  // read data file
+  if (!pReader->dataDone) {
+    code = tsdbSnapReadData(pReader, ppData);
+    if (code) {
+      if (code == TSDB_CODE_VND_READ_END) {
+        pReader->dataDone = 1;
+      } else {
+        goto _err;
+      }
+    } else {
+      goto _exit;
+    }
+  }
+
+  // read del file
+  if (!pReader->delDone) {
+    code = tsdbSnapReadDel(pReader, ppData);
+    if (code) {
+      if (code == TSDB_CODE_VND_READ_END) {
+        pReader->delDone = 1;
+      } else {
+        goto _err;
+      }
+    } else {
+      goto _exit;
+    }
+  }
+
+  code = TSDB_CODE_VND_READ_END;
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d snapshot read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+// STsdbSnapWriter ========================================
+struct STsdbSnapWriter {
+  STsdb*  pTsdb;
+  int64_t sver;
+  int64_t ever;
+
+  // config
+  int32_t minutes;
+  int8_t  precision;
+
+  // for data file
+  int32_t       fid;
+  SDataFReader* pDataFReader;
+  SArray*       aBlockIdx;
+  int32_t       iBlockIdx;
+  SBlockIdx*    pBlockIdx;
+  SMapData      mBlock;
+  int32_t       iBlock;
+  SBlockData    blockData;
+  int32_t       iRow;
+
+  SDataFWriter* pDataFWriter;
+  SArray*       aBlockIdxN;
+  SBlockIdx     blockIdx;
+  SMapData      mBlockN;
+  SBlock        block;
+  SBlockData    nBlockData;
+
+  // for del file
+  SDelFReader* pDelFReader;
+  SDelFWriter* pDelFWriter;
+  int32_t      iDelIdx;
+  SArray*      aDelIdx;
+  SArray*      aDelData;
+  SArray*      aDelIdxN;
+};
+
+static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDataFWriter == NULL) goto _exit;
+
+  // TODO
+
+  code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 0);
+  if (code) goto _err;
+
+  if (pWriter->pDataFReader) {
+    code = tsdbDataFReaderClose(&pWriter->pDataFReader);
+    if (code) goto _err;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t     code = 0;
+  int32_t     iRow = 0;           // todo
+  int32_t     nRow = 0;           // todo
+  SBlockData* pBlockData = NULL;  // todo
+
+  while (iRow < nRow) {
+    code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write append data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+  int64_t suid = 0;  // todo
+  int64_t uid = 0;   // todo
+  int64_t skey;      // todo
+  int64_t ekey;      // todo
+
+  int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision);
+  ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision));
+
+  // begin
+  if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
+
+    pWriter->fid = fid;
+    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid);
+    // reader
+    if (pSet) {
+      // open
+      code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
+      if (code) goto _err;
+
+      // SBlockIdx
+      code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL);
+      if (code) goto _err;
+    } else {
+      taosArrayClear(pWriter->aBlockIdx);
+    }
+    pWriter->iBlockIdx = 0;
+
+    // writer
+    SDFileSet wSet = {0};
+    if (pSet == NULL) {
+      wSet = (SDFileSet){0};  // todo
+    } else {
+      wSet = (SDFileSet){0};  // todo
+    }
+
+    code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
+    if (code) goto _err;
+
+    taosArrayClear(pWriter->aBlockIdxN);
+  }
+
+  // process
+  TABLEID id = {0};    // TODO
+  TSKEY   minKey = 0;  // TODO
+  TSKEY   maxKey = 0;  // TODO
+
+  while (true) {
+    if (pWriter->pBlockIdx) {
+      int32_t c = tTABLEIDCmprFn(&id, pWriter->pBlockIdx);
+
+      if (c == 0) {
+      } else if (c < 0) {
+        // keep merge
+      } else {
+        // code = tsdbSnapWriteTableDataEnd(pWriter);
+        if (code) goto _err;
+
+        pWriter->iBlockIdx++;
+        if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
+          pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
+        } else {
+          pWriter->pBlockIdx = NULL;
+        }
+
+        if (pWriter->pBlockIdx) {
+          code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL);
+          if (code) goto _err;
+        }
+      }
+    } else {
+      int32_t c = tTABLEIDCmprFn(&id, &pWriter->blockIdx);
+
+      if (c == 0) {
+        // merge commit the block data
+      } else if (c > 0) {
+        // code = tsdbSnapWriteTableDataEnd(pWriter);
+        if (code) goto _err;
+      } else {
+        ASSERT(0);
+      }
+    }
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDelFWriter == NULL) {
+    SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->nState);
+
+    // reader
+    if (pDelFile) {
+      code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL);
+      if (code) goto _err;
+
+      code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdx, NULL);
+      if (code) goto _err;
+    }
+
+    // writer
+    SDelFile delFile = {.commitID = pTsdb->pVnode->state.commitID, .offset = 0, .size = 0};
+    code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb);
+    if (code) goto _err;
+  }
+
+  // process the del data
+  TABLEID id = {0};  // todo
+
+  while (true) {
+    SDelIdx* pDelIdx = NULL;
+    int64_t  n = 0;
+    SDelData delData;
+    SDelIdx  delIdx;
+    int8_t   toBreak = 0;
+
+    if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx)) {
+      pDelIdx = taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+    }
+
+    if (pDelIdx) {
+      int32_t c = tTABLEIDCmprFn(&id, pDelIdx);
+      if (c < 0) {
+        goto _new_del;
+      } else {
+        code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
+        if (code) goto _err;
+
+        pWriter->iDelIdx++;
+        if (c == 0) {
+          toBreak = 1;
+          delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
+          goto _merge_del;
+        } else {
+          delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
+          goto _write_del;
+        }
+      }
+    }
+
+  _new_del:
+    toBreak = 1;
+    delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
+    taosArrayClear(pWriter->aDelData);
+
+  _merge_del:
+    while (n < nData) {
+      n += tGetDelData(pData + n, &delData);
+      if (taosArrayPush(pWriter->aDelData, &delData) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+    }
+
+  _write_del:
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
+    if (code) goto _err;
+
+    if (taosArrayPush(pWriter->aDelIdxN, &delIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+
+    if (toBreak) break;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDelFWriter == NULL) goto _exit;
+  for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx); pWriter->iDelIdx++) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+
+    code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
+    if (code) goto _err;
+
+    SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
+    if (code) goto _err;
+
+    if (taosArrayPush(pWriter->aDelIdx, &delIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+
+  code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
+  if (code) goto _err;
+
+  code = tsdbFSStateUpsertDelFile(pTsdb->fs->nState, &pWriter->pDelFWriter->fDel);
+  if (code) goto _err;
+
+  code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1);
+  if (code) goto _err;
+
+  if (pWriter->pDelFReader) {
+    code = tsdbDelFReaderClose(&pWriter->pDelFReader);
+    if (code) goto _err;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshow write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
+  int32_t          code = 0;
+  STsdbSnapWriter* pWriter = NULL;
+
+  // alloc
+  pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pTsdb = pTsdb;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  *ppWriter = pWriter;
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppWriter = NULL;
+  return code;
+}
+
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
+  int32_t          code = 0;
+  STsdbSnapWriter* pWriter = *ppWriter;
+
+  if (rollback) {
+    code = tsdbSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
+
+    code = tsdbSnapWriteDelEnd(pWriter);
+    if (code) goto _err;
+
+    code = tsdbSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+
+  taosMemoryFree(pWriter);
+  *ppWriter = NULL;
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  int8_t  type = pData[0];
+
+  // ts data
+  if (type == 0) {
+    code = tsdbSnapWriteData(pWriter, pData + 1, nData - 1);
+    if (code) goto _err;
+  } else {
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
+  }
+
+  // del data
+  if (type == 1) {
+    code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1);
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshow write failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
 }
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@ -87,8 +87,10 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {

  n += tPutI32v(p ? p + n : p, pMapData->nItem);
  if (pMapData->nItem) {
+    int32_t lOffset = 0;
    for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
-      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem]);
+      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset);
+      lOffset = pMapData->aOffset[iItem];
    }

    n += tPutI32v(p ? p + n : p, pMapData->nData);
@ -111,8 +113,11 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
  if (pMapData->nItem) {
    if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1;

+    int32_t lOffset = 0;
    for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
      n += tGetI32v(p + n, &pMapData->aOffset[iItem]);
+      pMapData->aOffset[iItem] += lOffset;
+      lOffset = pMapData->aOffset[iItem];
    }

    n += tGetI32v(p + n, &pMapData->nData);
--- a/source/dnode/vnode/src/vnd/vnodeCfg.c
+++ b/source/dnode/vnode/src/vnd/vnodeCfg.c
@ -15,30 +15,37 @@

 #include "vnd.h"

-const SVnodeCfg vnodeCfgDefault = {
-    .vgId = -1,
-    .dbname = "",
-    .dbId = 0,
-    .szPage = 4096,
-    .szCache = 256,
-    .szBuf = 96 * 1024 * 1024,
-    .isHeap = false,
-    .isWeak = 0,
-    .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
-                .update = 1,
-                .compression = 2,
-                .slLevel = 5,
-                .days = 14400,
-                .minRows = 100,
-                .maxRows = 4096,
-                .keep2 = 5256000,
-                .keep0 = 5256000,
-                .keep1 = 5256000},
-    .walCfg =
-        {.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE},
-    .hashBegin = 0,
-    .hashEnd = 0,
-    .hashMethod = 0};
+const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
+                                   .dbname = "",
+                                   .dbId = 0,
+                                   .szPage = 4096,
+                                   .szCache = 256,
+                                   .szBuf = 96 * 1024 * 1024,
+                                   .isHeap = false,
+                                   .isWeak = 0,
+                                   .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
+                                               .update = 1,
+                                               .compression = 2,
+                                               .slLevel = 5,
+                                               .days = 14400,
+                                               .minRows = 100,
+                                               .maxRows = 4096,
+                                               .keep2 = 5256000,
+                                               .keep0 = 5256000,
+                                               .keep1 = 5256000},
+                                   .walCfg =
+                                       {
+                                           .vgId = -1,
+                                           .fsyncPeriod = 0,
+                                           .retentionPeriod = -1,
+                                           .rollPeriod = -1,
+                                           .segSize = -1,
+                                           .retentionSize = -1,
+                                           .level = TAOS_WAL_WRITE,
+                                       },
+                                   .hashBegin = 0,
+                                   .hashEnd = 0,
+                                   .hashMethod = 0};

 int vnodeCheckCfg(const SVnodeCfg *pCfg) {
  // TODO
@ -79,7 +86,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
    SJson *pNodeRetentions = tjsonCreateArray();
    tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
    for (int32_t i = 0; i < nRetention; ++i) {
-      SJson *           pNodeRetention = tjsonCreateObject();
+      SJson            *pNodeRetention = tjsonCreateObject();
      const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
      tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq);
      tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit);
@ -156,7 +163,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
  if (code < 0) return -1;
  tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
  if (code < 0) return -1;
-  SJson * pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
+  SJson  *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
  int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
  if (nRetention > TSDB_RETENTION_MAX) {
    nRetention = TSDB_RETENTION_MAX;
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@ -15,7 +15,7 @@

 #include "vnd.h"

-#define VND_INFO_FNAME     "vnode.json"
+#define VND_INFO_FNAME "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"

 static int  vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
@ -223,12 +223,14 @@ int vnodeCommit(SVnode *pVnode) {
  // save info
  info.config = pVnode->config;
  info.state.committed = pVnode->state.applied;
+  info.state.commitTerm = pVnode->state.applyTerm;
  info.state.commitID = pVnode->state.commitID;
  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
  if (vnodeSaveInfo(dir, &info) < 0) {
    ASSERT(0);
    return -1;
  }
+  walBeginSnapshot(pVnode->pWal, pVnode->state.applied);

  // preCommit
  smaPreCommit(pVnode->pSma);
@ -270,13 +272,14 @@ int vnodeCommit(SVnode *pVnode) {
    ASSERT(0);
    return -1;
  }
-  
+
  pVnode->state.committed = info.state.committed;

  // postCommit
  smaPostCommit(pVnode->pSma);

  // apply the commit (TODO)
+  walEndSnapshot(pVnode->pWal);
  vnodeBufPoolReset(pVnode->onCommit);
  pVnode->onCommit->next = pVnode->pPool;
  pVnode->pPool = pVnode->onCommit;
@ -316,6 +319,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {

  if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
  if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1;

  return 0;
 }
@ -328,6 +332,8 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
  if (code < 0) return -1;
  tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
  if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
+  if (code < 0) return -1;

  return 0;
 }
--- a/source/dnode/vnode/src/vnd/vnodeModule.c
+++ b/source/dnode/vnode/src/vnd/vnodeModule.c
@ -100,6 +100,7 @@ void vnodeCleanup() {

  walCleanUp();
  tqCleanUp();
+  smaCleanUp();
 }

 int vnodeScheduleTask(int (*execute)(void*), void* arg) {
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@ -79,8 +79,10 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
  strcpy(pVnode->path, path);
  pVnode->config = info.config;
  pVnode->state.committed = info.state.committed;
+  pVnode->state.commitTerm = info.state.commitTerm;
  pVnode->state.applied = info.state.committed;
  pVnode->state.commitID = info.state.commitID;
+  pVnode->state.commitTerm = info.state.commitTerm;
  pVnode->pTfs = pTfs;
  pVnode->msgCb = msgCb;
  pVnode->blockCount = 0;
@ -115,6 +117,13 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
  // open wal
  sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR);
  taosRealPath(tdir, NULL, sizeof(tdir));
+
+// for test tsdb snapshot
+#if 0
+  pVnode->config.walCfg.segSize = 200;
+  pVnode->config.walCfg.retentionSize = 2000;
+#endif
+
  pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg));
  if (pVnode->pWal == NULL) {
    vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno));
@ -194,4 +203,9 @@ void vnodeStop(SVnode *pVnode) {}

 int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }

-void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }
+void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) {
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = pVnode->state.committed;
+  pSnapshot->lastApplyTerm = pVnode->state.commitTerm;
+  pSnapshot->lastConfigIndex = -1;
+}
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@ -13,24 +13,27 @@
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

-#include "vnodeInt.h"
+#include "vnd.h"

-struct SVSnapshotReader {
-  SVnode              *pVnode;
-  int64_t              sver;
-  int64_t              ever;
-  int8_t               isMetaEnd;
-  int8_t               isTsdbEnd;
-  SMetaSnapshotReader *pMetaReader;
-  STsdbSnapshotReader *pTsdbReader;
-  void                *pData;
-  int32_t              nData;
+// SVSnapReader ========================================================
+struct SVSnapReader {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+  // meta
+  int8_t           metaDone;
+  SMetaSnapReader *pMetaReader;
+  // tsdb
+  int8_t           tsdbDone;
+  STsdbSnapReader *pTsdbReader;
+  uint8_t         *pData;
 };

-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever) {
-  SVSnapshotReader *pReader = NULL;
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
+  int32_t       code = 0;
+  SVSnapReader *pReader = NULL;

-  pReader = (SVSnapshotReader *)taosMemoryCalloc(1, sizeof(*pReader));
+  pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
  if (pReader == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
@ -38,72 +41,169 @@ int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int
  pReader->pVnode = pVnode;
  pReader->sver = sver;
  pReader->ever = ever;
-  pReader->isMetaEnd = 0;
-  pReader->isTsdbEnd = 0;

-  if (metaSnapshotReaderOpen(pVnode->pMeta, &pReader->pMetaReader, sver, ever) < 0) {
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = metaSnapReaderOpen(pVnode->pMeta, sver, ever, &pReader->pMetaReader);
+  if (code) goto _err;

-  if (tsdbSnapshotReaderOpen(pVnode->pTsdb, &pReader->pTsdbReader, sver, ever) < 0) {
-    metaSnapshotReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = tsdbSnapReaderOpen(pVnode->pTsdb, sver, ever, &pReader->pTsdbReader);
+  if (code) goto _err;

-_exit:
  *ppReader = pReader;
-  return 0;
+  return code;

 _err:
+  vError("vgId:%d vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
  *ppReader = NULL;
-  return -1;
+  return code;
 }

-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader) {
-  if (pReader) {
-    vnodeFree(pReader->pData);
-    tsdbSnapshotReaderClose(pReader->pTsdbReader);
-    metaSnapshotReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-  }
-  return 0;
-}
-
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData) {
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
  int32_t code = 0;

-  if (!pReader->isMetaEnd) {
-    code = metaSnapshotRead(pReader->pMetaReader, &pReader->pData, &pReader->nData);
+  tFree(pReader->pData);
+  if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
+  if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
+  taosMemoryFree(pReader);
+
+  return code;
+}
+
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
+  int32_t code = 0;
+
+  if (!pReader->metaDone) {
+    code = metaSnapRead(pReader->pMetaReader, &pReader->pData);
    if (code) {
      if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isMetaEnd = 1;
+        pReader->metaDone = 1;
      } else {
-        return code;
+        goto _err;
      }
    } else {
      *ppData = pReader->pData;
-      *nData = pReader->nData;
-      return code;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
+      goto _exit;
    }
  }

-  if (!pReader->isTsdbEnd) {
-    code = tsdbSnapshotRead(pReader->pTsdbReader, &pReader->pData, &pReader->nData);
+  if (!pReader->tsdbDone) {
+    code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData);
    if (code) {
      if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isTsdbEnd = 1;
+        pReader->tsdbDone = 1;
      } else {
-        return code;
+        goto _err;
      }
    } else {
      *ppData = pReader->pData;
-      *nData = pReader->nData;
-      return code;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
+      goto _exit;
    }
  }

  code = TSDB_CODE_VND_READ_END;
+
+_exit:
+  return code;
+
+_err:
+  vError("vgId:% snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
+  return code;
+}
+
+// SVSnapWriter ========================================================
+struct SVSnapWriter {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+  // meta
+  SMetaSnapWriter *pMetaSnapWriter;
+  // tsdb
+  STsdbSnapWriter *pTsdbSnapWriter;
+};
+
+static int32_t vnodeSnapRollback(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t vnodeSnapCommit(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
+  int32_t       code = 0;
+  SVSnapWriter *pWriter = NULL;
+
+  // alloc
+  pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pVnode = pVnode;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
+  int32_t code = 0;
+
+  if (rollback) {
+    code = vnodeSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = vnodeSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+
+  taosMemoryFree(pWriter);
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshow writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
+  int32_t       code = 0;
+  SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData;
+  SVnode       *pVnode = pWriter->pVnode;
+
+  ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData);
+
+  if (pSnapDataHdr->type == 0) {
+    // meta
+    if (pWriter->pMetaSnapWriter == NULL) {
+      code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
+  } else {
+    // tsdb
+    if (pWriter->pTsdbSnapWriter == NULL) {
+      code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshot write failed since %s", TD_VID(pVnode), tstrerror(code));
  return code;
 }
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@ -24,7 +24,8 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
 static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);

@ -143,6 +144,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
         version);

  pVnode->state.applied = version;
+  pVnode->state.applyTerm = pMsg->info.conn.applyTerm;

  // skip header
  pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
@ -214,9 +216,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
      vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp);
      break;
    case TDMT_VND_ALTER_HASHRANGE:
-      vnodeProcessAlterHasnRangeReq(pVnode, version, pReq, len, pRsp);
+      vnodeProcessAlterHashRangeReq(pVnode, version, pReq, len, pRsp);
      break;
    case TDMT_VND_ALTER_CONFIG:
+      vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp);
      break;
    case TDMT_VND_COMMIT:
      goto _do_commit;
@ -885,7 +888,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void
  return 0;
 }

-static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
  vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode));

  // todo
@ -895,6 +898,18 @@ static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, vo
  return 0;
 }

+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  SAlterVnodeReq alterReq = {0};
+  if (tDeserializeSAlterVnodeReq(pReq, len, &alterReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return TSDB_CODE_INVALID_MSG;
+  }
+
+  vInfo("vgId:%d, start to alter vnode config, cacheLast:%d cacheLastSize:%d", TD_VID(pVnode), alterReq.cacheLast,
+        alterReq.cacheLastSize);
+  return 0;
+}
+
 static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
  int32_t     code = 0;
  SDecoder   *pCoder = &(SDecoder){0};
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@ -423,7 +423,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon

 static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
  SVnode *pVnode = pFsm->data;
-  vTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, msgtype:%d %s",
+  vTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s",
         syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state,
         syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType));

@ -438,43 +438,94 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c

 static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
  SVnode *pVnode = pFsm->data;
-  vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, msgtype:%d %s",
+  vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s",
         syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state,
         syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType));
 }

 static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
  SVnode *pVnode = pFsm->data;
-  vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, msgtype:%d %s",
+  vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s",
         syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state,
         syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType));
 }

+#define USE_TSDB_SNAPSHOT
+
 static int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void *pParam, void **ppReader) {
+#ifdef USE_TSDB_SNAPSHOT
  SVnode         *pVnode = pFsm->data;
  SSnapshotParam *pSnapshotParam = pParam;
-  int32_t         code =
-      vnodeSnapshotReaderOpen(pVnode, (SVSnapshotReader **)ppReader, pSnapshotParam->start, pSnapshotParam->end);
+  int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader);
  return code;
+#else
+  *ppReader = taosMemoryMalloc(32);
+  return 0;
+#endif
 }

 static int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) {
+#ifdef USE_TSDB_SNAPSHOT
  SVnode *pVnode = pFsm->data;
-  int32_t code = vnodeSnapshotReaderClose(pReader);
+  int32_t code = vnodeSnapReaderClose(pReader);
  return code;
+#else
+  taosMemoryFree(pReader);
+  return 0;
+#endif
 }

 static int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
+#ifdef USE_TSDB_SNAPSHOT
  SVnode *pVnode = pFsm->data;
-  int32_t code = vnodeSnapshotRead(pReader, (const void **)ppBuf, len);
+  int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
  return code;
+#else
+  static int32_t times = 0;
+  if (times++ < 5) {
+    *len = 64;
+    *ppBuf = taosMemoryMalloc(*len);
+    snprintf(*ppBuf, *len, "snapshot block %d", times);
+  } else {
+    *len = 0;
+    *ppBuf = NULL;
+  }
+  return 0;
+#endif
 }

-static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWriter) { return 0; }
+static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWriter) {
+#ifdef USE_TSDB_SNAPSHOT
+  SVnode         *pVnode = pFsm->data;
+  SSnapshotParam *pSnapshotParam = pParam;
+  int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter);
+  return code;
+#else
+  *ppWriter = taosMemoryMalloc(32);
+  return 0;
+#endif
+}

-static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { return 0; }
+static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) {
+#ifdef USE_TSDB_SNAPSHOT
+  SVnode *pVnode = pFsm->data;
+  int32_t code = vnodeSnapWriterClose(pWriter, !isApply);
+  return code;
+#else
+  taosMemoryFree(pWriter);
+  return 0;
+#endif
+}

-static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { return 0; }
+static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) {
+#ifdef USE_TSDB_SNAPSHOT
+  SVnode *pVnode = pFsm->data;
+  int32_t code = vnodeSnapWrite(pWriter, pBuf, len);
+  return code;
+#else
+  return 0;
+#endif
+}

 static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
  SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
@ -497,7 +548,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {

 int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
  SSyncInfo syncInfo = {
-      .snapshotStrategy = SYNC_STRATEGY_NO_SNAPSHOT,
+      .snapshotStrategy = SYNC_STRATEGY_WAL_FIRST,
+      //.snapshotStrategy = SYNC_STRATEGY_NO_SNAPSHOT,
      .batchSize = 10,
      .vgId = pVnode->config.vgId,
      .isStandBy = pVnode->config.standby,
--- a/source/libs/catalog/inc/catalogInt.h
+++ b/source/libs/catalog/inc/catalogInt.h
@ -166,7 +166,7 @@ typedef struct SCtgDBCache {
  int8_t           deleted;
  SCtgVgCache      vgCache;
  SHashObj        *tbCache;         // key:tbname, value:SCtgTbCache
-  SHashObj        *stbCache;        // key:suid, value:STableMeta*
+  SHashObj        *stbCache;        // key:suid, value:char*
 } SCtgDBCache;

 typedef struct SCtgRentSlot {
@ -480,37 +480,35 @@ typedef struct SCtgOperation {

 #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000

-#define CTG_IS_LOCKED(_lock) atomic_load_32((_lock))
-
 #define CTG_LOCK(type, _lock) do {   \
  if (CTG_READ == (type)) {          \
-    assert(atomic_load_32((_lock)) >= 0);  \
-    CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG RLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
    taosRLockLatch(_lock);           \
-    CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
-    assert(atomic_load_32((_lock)) > 0);  \
+    CTG_LOCK_DEBUG("CTG RLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) > 0);  \
  } else {                                                \
-    assert(atomic_load_32((_lock)) >= 0);  \
-    CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
+    assert(atomic_load_64((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG WLOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__);  \
    taosWLockLatch(_lock);                                \
-    CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
-    assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
+    CTG_LOCK_DEBUG("CTG WLOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__);  \
+    assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
  }                                                       \
 } while (0)

 #define CTG_UNLOCK(type, _lock) do {                       \
  if (CTG_READ == (type)) {                                \
-    assert(atomic_load_32((_lock)) > 0);  \
-    CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) > 0);  \
+    CTG_LOCK_DEBUG("CTG RULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
    taosRUnLockLatch(_lock);                              \
-    CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
-    assert(atomic_load_32((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG RULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) >= 0);  \
  } else {                                                \
-    assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
-    CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
+    CTG_LOCK_DEBUG("CTG WULOCK%p:%" PRIx64 ", %s:%d B", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
    taosWUnLockLatch(_lock);                              \
-    CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
-    assert(atomic_load_32((_lock)) >= 0);  \
+    CTG_LOCK_DEBUG("CTG WULOCK%p:%" PRIx64 ", %s:%d E", (_lock), atomic_load_64(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_64((_lock)) >= 0);  \
  }                                                       \
 } while (0)

--- a/source/libs/catalog/src/ctgAsync.c
+++ b/source/libs/catalog/src/ctgAsync.c
@ -789,9 +789,13 @@ _return:

 int32_t ctgCallUserCb(void* param) {
  SCtgJob* pJob = (SCtgJob*)param;
+
+  qDebug("QID:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode));
  
  (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode);

+  qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId);
+
  taosRemoveRef(gCtgMgmt.jobPool, pJob->refId);

  return TSDB_CODE_SUCCESS;
@ -822,8 +826,6 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {

 _return:

-  qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(code));
-
  pJob->jobResCode = code;

  //taosSsleep(2);
--- a/Show More
+++ b/Show More