Merge pull request #20320 from taosdata/fix/3.0_merge_main

merge main

docs/zh/14-reference/07-tdinsight/index.mdx

@@ -140,10 +140,10 @@ Install and configure TDinsight dashboard in Grafana on Ubuntu 18.04/20.04 syste
 | -n     | --tdengine-ds-name         | TDENGINE_DS_NAME             | TDengine 数据源名称，默认为 TDengine。                  |
 | -a     | --tdengine-api             | TDENGINE_API                 | TDengine REST API 端点。默认为`http://127.0.0.1:6041`。 |
 | -u     | --tdengine-user            | TDENGINE_USER                | TDengine 用户名。 [默认值：root]                        |
-| -p     | --tdengine-密码            | TDENGINE_PASSWORD            | TDengine 密码。 [默认：taosdata]                        |
+| -p     | --tdengine-password        | TDENGINE_PASSWORD            | TDengine 密码。 [默认：taosdata]                        |
 | -i     | --tdinsight-uid            | TDINSIGHT_DASHBOARD_UID      | TDinsight 仪表盘`uid`。 [默认值：tdinsight]             |
 | -t     | --tdinsight-title          | TDINSIGHT_DASHBOARD_TITLE    | TDinsight 仪表盘标题。 [默认：TDinsight]                |
-| -e     | --tdinsight-可编辑         | TDINSIGHT_DASHBOARD_EDITABLE | 如果配置仪表盘可以编辑。 [默认值：false]                |
+| -e     | --tdinsight-editable       | TDINSIGHT_DASHBOARD_EDITABLE | 如果配置仪表盘可以编辑。 [默认值：false]                |
 | -E     | --external-notifier        | EXTERNAL_NOTIFIER            | 将外部通知程序 uid 应用于 TDinsight 仪表盘。            |
 
 假设您在主机 `tdengine` 上启动 TDengine 数据库，HTTP API 端口为 `6041`，用户为 `root1`，密码为 `pass5ord`。执行脚本：

include/common/tglobal.h

@@ -74,6 +74,10 @@ extern int32_t tsHeartbeatTimeout;
 // vnode
 extern int64_t tsVndCommitMaxIntervalMs;
 
+// mnode
+extern int64_t tsMndSdbWriteDelta;
+extern int64_t tsMndLogRetention;
+
 // monitor
 extern bool     tsEnableMonitor;
 extern int32_t  tsMonitorInterval;

include/common/tmsg.h

@@ -66,6 +66,15 @@ extern int32_t tMsgDict[];
 
 typedef uint16_t tmsg_t;
 
+static inline bool tmsgIsValid(tmsg_t type) {
+  if (type < TDMT_DND_MAX_MSG || type < TDMT_MND_MAX_MSG || type < TDMT_VND_MAX_MSG || type < TDMT_SCH_MAX_MSG ||
+      type < TDMT_STREAM_MAX_MSG || type < TDMT_MON_MAX_MSG || type < TDMT_SYNC_MAX_MSG || type < TDMT_VND_STREAM_MSG ||
+      type < TDMT_VND_TMQ_MSG || type < TDMT_VND_TMQ_MAX_MSG) {
+    return true;
+  } else {
+    return false;
+  }
+}
 static inline bool vnodeIsMsgBlock(tmsg_t type) {
   return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) ||
          (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM);
@@ -603,6 +612,7 @@ typedef struct {
   char    user[TSDB_USER_LEN];
   char    passwd[TSDB_PASSWORD_LEN];
   int64_t startTime;
+  char    sVer[TSDB_VERSION_LEN];
 } SConnectReq;
 
 int32_t tSerializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq);
@@ -929,12 +939,19 @@ typedef struct {
   int32_t minRows;
   int32_t maxRows;
   int32_t walFsyncPeriod;
+  int16_t hashPrefix;
+  int16_t hashSuffix;
   int8_t  walLevel;
   int8_t  precision;
   int8_t  compression;
   int8_t  replications;
   int8_t  strict;
   int8_t  cacheLast;
+  int32_t tsdbPageSize;
+  int32_t walRetentionPeriod;
+  int32_t walRollPeriod;
+  int64_t walRetentionSize;
+  int64_t walSegmentSize;
   int32_t numOfRetensions;
   SArray* pRetensions;
   int8_t  schemaless;
@@ -1015,7 +1032,8 @@ int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchR
 void    tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp);
 
 typedef struct {
-  char db[TSDB_DB_FNAME_LEN];
+  char        db[TSDB_DB_FNAME_LEN];
+  STimeWindow timeRange;
 } SCompactDbReq;
 
 int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
@@ -1274,9 +1292,10 @@ int32_t tSerializeSDropIdxReq(void* buf, int32_t bufLen, SDropIndexReq* pReq);
 int32_t tDeserializeSDropIdxReq(void* buf, int32_t bufLen, SDropIndexReq* pReq);
 
 typedef struct {
-  int64_t dbUid;
+  int64_t     dbUid;
-  char    db[TSDB_DB_FNAME_LEN];
+  char        db[TSDB_DB_FNAME_LEN];
-  int64_t compactStartTime;
+  int64_t     compactStartTime;
+  STimeWindow tw;
 } SCompactVnodeReq;
 
 int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
@@ -1812,7 +1831,7 @@ typedef struct {
 #define STREAM_TRIGGER_AT_ONCE        1
 #define STREAM_TRIGGER_WINDOW_CLOSE   2
 #define STREAM_TRIGGER_MAX_DELAY      3
-#define STREAM_DEFAULT_IGNORE_EXPIRED 0
+#define STREAM_DEFAULT_IGNORE_EXPIRED 1
 #define STREAM_FILL_HISTORY_ON        1
 #define STREAM_FILL_HISTORY_OFF       0
 #define STREAM_DEFAULT_FILL_HISTORY   STREAM_FILL_HISTORY_OFF
@@ -1904,10 +1923,10 @@ typedef struct {
 } SMqConsumerLostMsg, SMqConsumerRecoverMsg, SMqConsumerClearMsg;
 
 typedef struct {
-  int64_t  consumerId;
+  int64_t consumerId;
-  char     cgroup[TSDB_CGROUP_LEN];
+  char    cgroup[TSDB_CGROUP_LEN];
-  char     clientId[256];
+  char    clientId[256];
-  SArray*  topicNames;  // SArray<char**>
+  SArray* topicNames;  // SArray<char**>
 } SCMSubscribeReq;
 
 static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
@@ -2684,7 +2703,7 @@ typedef struct {
   char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
   int8_t  subType;
   int8_t  withMeta;
-  char*   qmsg;
+  char*   qmsg;  // SubPlanToString
   int64_t suid;
 } SMqRebVgReq;
 

include/common/ttokendef.h

@@ -110,169 +110,169 @@
 #define TK_TABLE_SUFFIX         92
 #define TK_NK_COLON             93
 #define TK_MAX_SPEED            94
-#define TK_TABLE                95
+#define TK_START                95
-#define TK_NK_LP                96
+#define TK_WITH                 96
-#define TK_NK_RP                97
+#define TK_TIMESTAMP            97
-#define TK_STABLE               98
+#define TK_END                  98
-#define TK_ADD                  99
+#define TK_TABLE                99
-#define TK_COLUMN               100
+#define TK_NK_LP                100
-#define TK_MODIFY               101
+#define TK_NK_RP                101
-#define TK_RENAME               102
+#define TK_STABLE               102
-#define TK_TAG                  103
+#define TK_ADD                  103
-#define TK_SET                  104
+#define TK_COLUMN               104
-#define TK_NK_EQ                105
+#define TK_MODIFY               105
-#define TK_USING                106
+#define TK_RENAME               106
-#define TK_TAGS                 107
+#define TK_TAG                  107
-#define TK_COMMENT              108
+#define TK_SET                  108
-#define TK_BOOL                 109
+#define TK_NK_EQ                109
-#define TK_TINYINT              110
+#define TK_USING                110
-#define TK_SMALLINT             111
+#define TK_TAGS                 111
-#define TK_INT                  112
+#define TK_COMMENT              112
-#define TK_INTEGER              113
+#define TK_BOOL                 113
-#define TK_BIGINT               114
+#define TK_TINYINT              114
-#define TK_FLOAT                115
+#define TK_SMALLINT             115
-#define TK_DOUBLE               116
+#define TK_INT                  116
-#define TK_BINARY               117
+#define TK_INTEGER              117
-#define TK_TIMESTAMP            118
+#define TK_BIGINT               118
-#define TK_NCHAR                119
+#define TK_FLOAT                119
-#define TK_UNSIGNED             120
+#define TK_DOUBLE               120
-#define TK_JSON                 121
+#define TK_BINARY               121
-#define TK_VARCHAR              122
+#define TK_NCHAR                122
-#define TK_MEDIUMBLOB           123
+#define TK_UNSIGNED             123
-#define TK_BLOB                 124
+#define TK_JSON                 124
-#define TK_VARBINARY            125
+#define TK_VARCHAR              125
-#define TK_DECIMAL              126
+#define TK_MEDIUMBLOB           126
-#define TK_MAX_DELAY            127
+#define TK_BLOB                 127
-#define TK_WATERMARK            128
+#define TK_VARBINARY            128
-#define TK_ROLLUP               129
+#define TK_DECIMAL              129
-#define TK_TTL                  130
+#define TK_MAX_DELAY            130
-#define TK_SMA                  131
+#define TK_WATERMARK            131
-#define TK_DELETE_MARK          132
+#define TK_ROLLUP               132
-#define TK_FIRST                133
+#define TK_TTL                  133
-#define TK_LAST                 134
+#define TK_SMA                  134
-#define TK_SHOW                 135
+#define TK_DELETE_MARK          135
-#define TK_PRIVILEGES           136
+#define TK_FIRST                136
-#define TK_DATABASES            137
+#define TK_LAST                 137
-#define TK_TABLES               138
+#define TK_SHOW                 138
-#define TK_STABLES              139
+#define TK_PRIVILEGES           139
-#define TK_MNODES               140
+#define TK_DATABASES            140
-#define TK_QNODES               141
+#define TK_TABLES               141
-#define TK_FUNCTIONS            142
+#define TK_STABLES              142
-#define TK_INDEXES              143
+#define TK_MNODES               143
-#define TK_ACCOUNTS             144
+#define TK_QNODES               144
-#define TK_APPS                 145
+#define TK_FUNCTIONS            145
-#define TK_CONNECTIONS          146
+#define TK_INDEXES              146
-#define TK_LICENCES             147
+#define TK_ACCOUNTS             147
-#define TK_GRANTS               148
+#define TK_APPS                 148
-#define TK_QUERIES              149
+#define TK_CONNECTIONS          149
-#define TK_SCORES               150
+#define TK_LICENCES             150
-#define TK_TOPICS               151
+#define TK_GRANTS               151
-#define TK_VARIABLES            152
+#define TK_QUERIES              152
-#define TK_CLUSTER              153
+#define TK_SCORES               153
-#define TK_BNODES               154
+#define TK_TOPICS               154
-#define TK_SNODES               155
+#define TK_VARIABLES            155
-#define TK_TRANSACTIONS         156
+#define TK_CLUSTER              156
-#define TK_DISTRIBUTED          157
+#define TK_BNODES               157
-#define TK_CONSUMERS            158
+#define TK_SNODES               158
-#define TK_SUBSCRIPTIONS        159
+#define TK_TRANSACTIONS         159
-#define TK_VNODES               160
+#define TK_DISTRIBUTED          160
-#define TK_ALIVE                161
+#define TK_CONSUMERS            161
-#define TK_LIKE                 162
+#define TK_SUBSCRIPTIONS        162
-#define TK_TBNAME               163
+#define TK_VNODES               163
-#define TK_QTAGS                164
+#define TK_ALIVE                164
-#define TK_AS                   165
+#define TK_LIKE                 165
-#define TK_INDEX                166
+#define TK_TBNAME               166
-#define TK_FUNCTION             167
+#define TK_QTAGS                167
-#define TK_INTERVAL             168
+#define TK_AS                   168
-#define TK_COUNT                169
+#define TK_INDEX                169
-#define TK_LAST_ROW             170
+#define TK_FUNCTION             170
-#define TK_TOPIC                171
+#define TK_INTERVAL             171
-#define TK_WITH                 172
+#define TK_COUNT                172
-#define TK_META                 173
+#define TK_LAST_ROW             173
-#define TK_CONSUMER             174
+#define TK_TOPIC                174
-#define TK_GROUP                175
+#define TK_META                 175
-#define TK_DESC                 176
+#define TK_CONSUMER             176
-#define TK_DESCRIBE             177
+#define TK_GROUP                177
-#define TK_RESET                178
+#define TK_DESC                 178
-#define TK_QUERY                179
+#define TK_DESCRIBE             179
-#define TK_CACHE                180
+#define TK_RESET                180
-#define TK_EXPLAIN              181
+#define TK_QUERY                181
-#define TK_ANALYZE              182
+#define TK_CACHE                182
-#define TK_VERBOSE              183
+#define TK_EXPLAIN              183
-#define TK_NK_BOOL              184
+#define TK_ANALYZE              184
-#define TK_RATIO                185
+#define TK_VERBOSE              185
-#define TK_NK_FLOAT             186
+#define TK_NK_BOOL              186
-#define TK_OUTPUTTYPE           187
+#define TK_RATIO                187
-#define TK_AGGREGATE            188
+#define TK_NK_FLOAT             188
-#define TK_BUFSIZE              189
+#define TK_OUTPUTTYPE           189
-#define TK_LANGUAGE             190
+#define TK_AGGREGATE            190
-#define TK_STREAM               191
+#define TK_BUFSIZE              191
-#define TK_INTO                 192
+#define TK_LANGUAGE             192
-#define TK_TRIGGER              193
+#define TK_STREAM               193
-#define TK_AT_ONCE              194
+#define TK_INTO                 194
-#define TK_WINDOW_CLOSE         195
+#define TK_TRIGGER              195
-#define TK_IGNORE               196
+#define TK_AT_ONCE              196
-#define TK_EXPIRED              197
+#define TK_WINDOW_CLOSE         197
-#define TK_FILL_HISTORY         198
+#define TK_IGNORE               198
-#define TK_UPDATE               199
+#define TK_EXPIRED              199
-#define TK_SUBTABLE             200
+#define TK_FILL_HISTORY         200
-#define TK_KILL                 201
+#define TK_UPDATE               201
-#define TK_CONNECTION           202
+#define TK_SUBTABLE             202
-#define TK_TRANSACTION          203
+#define TK_KILL                 203
-#define TK_BALANCE              204
+#define TK_CONNECTION           204
-#define TK_VGROUP               205
+#define TK_TRANSACTION          205
-#define TK_MERGE                206
+#define TK_BALANCE              206
-#define TK_REDISTRIBUTE         207
+#define TK_VGROUP               207
-#define TK_SPLIT                208
+#define TK_MERGE                208
-#define TK_DELETE               209
+#define TK_REDISTRIBUTE         209
-#define TK_INSERT               210
+#define TK_SPLIT                210
-#define TK_NULL                 211
+#define TK_DELETE               211
-#define TK_NK_QUESTION          212
+#define TK_INSERT               212
-#define TK_NK_ARROW             213
+#define TK_NULL                 213
-#define TK_ROWTS                214
+#define TK_NK_QUESTION          214
-#define TK_QSTART               215
+#define TK_NK_ARROW             215
-#define TK_QEND                 216
+#define TK_ROWTS                216
-#define TK_QDURATION            217
+#define TK_QSTART               217
-#define TK_WSTART               218
+#define TK_QEND                 218
-#define TK_WEND                 219
+#define TK_QDURATION            219
-#define TK_WDURATION            220
+#define TK_WSTART               220
-#define TK_IROWTS               221
+#define TK_WEND                 221
-#define TK_ISFILLED             222
+#define TK_WDURATION            222
-#define TK_CAST                 223
+#define TK_IROWTS               223
-#define TK_NOW                  224
+#define TK_ISFILLED             224
-#define TK_TODAY                225
+#define TK_CAST                 225
-#define TK_TIMEZONE             226
+#define TK_NOW                  226
-#define TK_CLIENT_VERSION       227
+#define TK_TODAY                227
-#define TK_SERVER_VERSION       228
+#define TK_TIMEZONE             228
-#define TK_SERVER_STATUS        229
+#define TK_CLIENT_VERSION       229
-#define TK_CURRENT_USER         230
+#define TK_SERVER_VERSION       230
-#define TK_CASE                 231
+#define TK_SERVER_STATUS        231
-#define TK_END                  232
+#define TK_CURRENT_USER         232
-#define TK_WHEN                 233
+#define TK_CASE                 233
-#define TK_THEN                 234
+#define TK_WHEN                 234
-#define TK_ELSE                 235
+#define TK_THEN                 235
-#define TK_BETWEEN              236
+#define TK_ELSE                 236
-#define TK_IS                   237
+#define TK_BETWEEN              237
-#define TK_NK_LT                238
+#define TK_IS                   238
-#define TK_NK_GT                239
+#define TK_NK_LT                239
-#define TK_NK_LE                240
+#define TK_NK_GT                240
-#define TK_NK_GE                241
+#define TK_NK_LE                241
-#define TK_NK_NE                242
+#define TK_NK_GE                242
-#define TK_MATCH                243
+#define TK_NK_NE                243
-#define TK_NMATCH               244
+#define TK_MATCH                244
-#define TK_CONTAINS             245
+#define TK_NMATCH               245
-#define TK_IN                   246
+#define TK_CONTAINS             246
-#define TK_JOIN                 247
+#define TK_IN                   247
-#define TK_INNER                248
+#define TK_JOIN                 248
-#define TK_SELECT               249
+#define TK_INNER                249
-#define TK_DISTINCT             250
+#define TK_SELECT               250
-#define TK_WHERE                251
+#define TK_DISTINCT             251
-#define TK_PARTITION            252
+#define TK_WHERE                252
-#define TK_BY                   253
+#define TK_PARTITION            253
-#define TK_SESSION              254
+#define TK_BY                   254
-#define TK_STATE_WINDOW         255
+#define TK_SESSION              255
-#define TK_EVENT_WINDOW         256
+#define TK_STATE_WINDOW         256
-#define TK_START                257
+#define TK_EVENT_WINDOW         257
 #define TK_SLIDING              258
 #define TK_FILL                 259
 #define TK_VALUE                260

include/libs/executor/executor.h

@@ -219,6 +219,7 @@ int32_t qStreamRecoverFinish(qTaskInfo_t tinfo);
 int32_t qStreamRestoreParam(qTaskInfo_t tinfo);
 bool    qStreamRecoverScanFinished(qTaskInfo_t tinfo);
 void    qStreamCloseTsdbReader(void* task);
+void    resetTaskInfo(qTaskInfo_t tinfo);
 
 #ifdef __cplusplus
 }

include/libs/nodes/cmdnodes.h

@@ -41,14 +41,15 @@ extern "C" {
 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
 
 #define SHOW_ALIVE_RESULT_COLS 1
-#define PRIVILEGE_TYPE_MASK(n) (1 << n)
 
-#define PRIVILEGE_TYPE_ALL       PRIVILEGE_TYPE_MASK(0)
+#define BIT_FLAG_MASK(n)              (1 << n)
-#define PRIVILEGE_TYPE_READ      PRIVILEGE_TYPE_MASK(1)
+#define BIT_FLAG_SET_MASK(val, mask)  ((val) |= (mask))
-#define PRIVILEGE_TYPE_WRITE     PRIVILEGE_TYPE_MASK(2)
+#define BIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
-#define PRIVILEGE_TYPE_SUBSCRIBE PRIVILEGE_TYPE_MASK(3)
 
-#define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0)
+#define PRIVILEGE_TYPE_ALL       BIT_FLAG_MASK(0)
+#define PRIVILEGE_TYPE_READ      BIT_FLAG_MASK(1)
+#define PRIVILEGE_TYPE_WRITE     BIT_FLAG_MASK(2)
+#define PRIVILEGE_TYPE_SUBSCRIBE BIT_FLAG_MASK(3)
 
 typedef struct SDatabaseOptions {
   ENodeType   type;
@@ -127,6 +128,8 @@ typedef struct STrimDatabaseStmt {
 typedef struct SCompactDatabaseStmt {
   ENodeType type;
   char      dbName[TSDB_DB_NAME_LEN];
+  SNode*    pStart;
+  SNode*    pEnd;
 } SCompactDatabaseStmt;
 
 typedef struct STableOptions {
@@ -265,6 +268,7 @@ typedef struct SShowStmt {
 typedef struct SShowCreateDatabaseStmt {
   ENodeType type;
   char      dbName[TSDB_DB_NAME_LEN];
+  char      dbFName[TSDB_DB_FNAME_LEN];
   void*     pCfg;  // SDbCfgInfo
 } SShowCreateDatabaseStmt;
 
@@ -392,6 +396,15 @@ typedef struct SKillQueryStmt {
   char      queryId[TSDB_QUERY_ID_LEN];
 } SKillQueryStmt;
 
+typedef enum EStreamOptionsSetFlag {
+  SOPT_TRIGGER_TYPE_SET = BIT_FLAG_MASK(0),
+  SOPT_WATERMARK_SET = BIT_FLAG_MASK(1),
+  SOPT_DELETE_MARK_SET = BIT_FLAG_MASK(2),
+  SOPT_FILL_HISTORY_SET = BIT_FLAG_MASK(3),
+  SOPT_IGNORE_EXPIRED_SET = BIT_FLAG_MASK(4),
+  SOPT_IGNORE_UPDATE_SET = BIT_FLAG_MASK(5),
+} EStreamOptionsSetFlag;
+
 typedef struct SStreamOptions {
   ENodeType type;
   int8_t    triggerType;
@@ -401,6 +414,7 @@ typedef struct SStreamOptions {
   int8_t    fillHistory;
   int8_t    ignoreExpired;
   int8_t    ignoreUpdate;
+  int64_t   setFlag;
 } SStreamOptions;
 
 typedef struct SCreateStreamStmt {

include/libs/sync/sync.h

@@ -35,7 +35,6 @@ extern "C" {
 #define SYNC_MAX_RECV_TIME_RANGE_MS  1200
 #define SYNC_DEL_WAL_MS              (1000 * 60)
 #define SYNC_ADD_QUORUM_COUNT        3
-#define SYNC_MNODE_LOG_RETENTION     10000
 #define SYNC_VNODE_LOG_RETENTION     (TSDB_SYNC_LOG_BUFFER_RETENTION + 1)
 #define SNAPSHOT_MAX_CLOCK_SKEW_MS   1000 * 10
 #define SNAPSHOT_WAIT_MS             1000 * 30

include/util/taoserror.h

@@ -116,6 +116,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_NO_DISKSPACE                  TAOS_DEF_ERROR_CODE(0, 0x012B)
 #define TSDB_CODE_TIMEOUT_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x012C)
 #define TSDB_CODE_MSG_ENCODE_ERROR              TAOS_DEF_ERROR_CODE(0, 0x012D)
+#define TSDB_CODE_NO_ENOUGH_DISKSPACE           TAOS_DEF_ERROR_CODE(0, 0x012E)
 
 #define TSDB_CODE_APP_IS_STARTING               TAOS_DEF_ERROR_CODE(0, 0x0130) //
 #define TSDB_CODE_APP_IS_STOPPING               TAOS_DEF_ERROR_CODE(0, 0x0131) //

include/util/tversion.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_UTIL_VERSION_H_
+#ifndef _TD_UTIL_TVERSION_H_
-#define _TD_UTIL_VERSION_H_
+#define _TD_UTIL_TVERSION_H_
 
 #include "os.h"
 
@@ -25,9 +25,11 @@ extern "C" {
 int32_t taosVersionStrToInt(const char *vstr, int32_t *vint);
 int32_t taosVersionIntToStr(int32_t vint, char *vstr, int32_t len);
 int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t comparedSegments);
+int32_t taosCheckVersionCompatibleFromStr(const char *pClientVersion, const char *pServerVersion,
+                                          int32_t comparedSegments);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_UTIL_VERSION_H_*/
+#endif /*_TD_UTIL_TVERSION_H_*/

packaging/tools/install.sh

@@ -34,6 +34,7 @@ benchmarkName="taosBenchmark"
 dumpName="taosdump"
 demoName="taosdemo"
 xname="taosx"
+explorerName="${clientName}-explorer"
 
 clientName2="taos"
 serverName2="taosd"
@@ -214,6 +215,7 @@ function install_bin() {
   ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || :
   ${csudo}rm -f ${bin_link_dir}/${dumpName} || :
   ${csudo}rm -f ${bin_link_dir}/${xname} || :
+  ${csudo}rm -f ${bin_link_dir}/${explorerName} || :
   ${csudo}rm -f ${bin_link_dir}/set_core || :
   ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
 
@@ -228,6 +230,7 @@ function install_bin() {
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
   [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
   [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || :
+  [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -s ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || :
   [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
   [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
   [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

packaging/tools/make_install.bat

@@ -70,6 +70,9 @@ if %Enterprise% == TRUE (
     if exist %binary_dir%\\build\\bin\\taosx.exe (
         copy %binary_dir%\\build\\bin\\taosx.exe %target_dir% > nul
     )
+    if exist %binary_dir%\\build\\bin\\taos-explorer.exe (
+        copy %binary_dir%\\build\\bin\\taos-explorer.exe %target_dir% > nul
+    )
     if exist %binary_dir%\\build\\bin\\tmq_sim.exe (
         copy %binary_dir%\\build\\bin\\tmq_sim.exe %target_dir% > nul
     )

packaging/tools/makepkg.sh

@@ -89,11 +89,13 @@ else
       ${build_dir}/bin/tdengine-datasource.zip \
       ${build_dir}/bin/tdengine-datasource.zip.md5sum"
   [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx"
+  explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer')
 
   bin_files="${build_dir}/bin/${serverName} \
       ${build_dir}/bin/${clientName} \
       ${taostools_bin_files} \
       ${taosx_bin} \
+      ${explorer_bin_files} \
       ${build_dir}/bin/taosadapter \
       ${build_dir}/bin/udfd \
       ${script_dir}/remove.sh \

packaging/tools/remove.sh

@@ -119,6 +119,7 @@ function clean_bin() {
   ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
   ${csudo}rm -f ${bin_link_dir}/taoskeeper || :
   ${csudo}rm -f ${bin_link_dir}/taosx || :
+  ${csudo}rm -f ${bin_link_dir}/taos-explorer || :
 
   if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
     ${csudo}rm -f ${bin_link_dir}/${clientName2} || :

source/client/inc/clientInt.h

@@ -66,7 +66,8 @@ enum {
 typedef struct SAppInstInfo SAppInstInfo;
 
 typedef struct {
-  char* key;
+  char*   key;
+  int32_t idx;
   // statistics
   int32_t reportCnt;
   int32_t connKeyCnt;

source/client/inc/clientSml.h

@@ -190,7 +190,7 @@ typedef struct {
   //
   SArray      *preLineTagKV;
   SArray      *maxTagKVs;
-  SArray      *preLineColKV;
+  SArray      *masColKVs;
 
   SSmlLineInfo preLine;
   STableMeta  *currSTableMeta;

source/client/src/clientEnv.c

@@ -159,9 +159,9 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
   rpcInit.retryMaxInterval = tsRedirectMaxPeriod;
   rpcInit.retryMaxTimouet = tsMaxRetryWaitTime;
 
-  int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 5);
+  int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3);
   connLimitNum = TMAX(connLimitNum, 10);
-  connLimitNum = TMIN(connLimitNum, 500);
+  connLimitNum = TMIN(connLimitNum, 1000);
   rpcInit.connLimitNum = connLimitNum;
   rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
 

source/client/src/clientHb.c

@@ -303,8 +303,12 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
 }
 
 static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
+  if (0 == atomic_load_8(&clientHbMgr.inited)) {
+    goto _return;
+  }
+
   static int32_t    emptyRspNum = 0;
-  char             *key = (char *)param;
+  int32_t idx = *(int32_t *)param;
   SClientHbBatchRsp pRsp = {0};
   if (TSDB_CODE_SUCCESS == code) {
     tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
@@ -319,22 +323,24 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
 
   int32_t rspNum = taosArrayGetSize(pRsp.rsps);
 
-  taosThreadMutexLock(&appInfo.mutex);
+  taosThreadMutexLock(&clientHbMgr.lock);
 
-  SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
+  SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, idx);
-  if (pInst == NULL || NULL == *pInst) {
+  if (pAppHbMgr == NULL) {
-    taosThreadMutexUnlock(&appInfo.mutex);
+    taosThreadMutexUnlock(&clientHbMgr.lock);
-    tscError("cluster not exist, key:%s", key);
+    tscError("appHbMgr not exist, idx:%d", idx);
     taosMemoryFree(pMsg->pData);
     taosMemoryFree(pMsg->pEpSet);
     tFreeClientHbBatchRsp(&pRsp);
     return -1;
   }
 
+  SAppInstInfo *pInst = pAppHbMgr->pAppInstInfo;
+
   if (code != 0) {
-    (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1);
+    pInst->onlineDnodes = pInst->totalDnodes ? 0 : -1;
-    tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes,
+    tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), pInst->onlineDnodes,
-             (*pInst)->totalDnodes);
+             pInst->totalDnodes);
   }
 
   if (rspNum) {
@@ -346,15 +352,17 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
 
   for (int32_t i = 0; i < rspNum; ++i) {
     SClientHbRsp *rsp = taosArrayGet(pRsp.rsps, i);
-    code = (*clientHbMgr.rspHandle[rsp->connKey.connType])((*pInst)->pAppHbMgr, rsp);
+    code = (*clientHbMgr.rspHandle[rsp->connKey.connType])(pAppHbMgr, rsp);
     if (code) {
       break;
     }
   }
 
-  taosThreadMutexUnlock(&appInfo.mutex);
+  taosThreadMutexUnlock(&clientHbMgr.lock);
 
   tFreeClientHbBatchRsp(&pRsp);
+
+_return:
   taosMemoryFree(pMsg->pData);
   taosMemoryFree(pMsg->pEpSet);
   return code;
@@ -788,7 +796,8 @@ static void *hbThreadFunc(void *param) {
       pInfo->msgInfo.pData = buf;
       pInfo->msgInfo.len = tlen;
       pInfo->msgType = TDMT_MND_HEARTBEAT;
-      pInfo->param = taosStrdup(pAppHbMgr->key);
+      pInfo->param = taosMemoryMalloc(sizeof(int32_t));
+      *(int32_t *)pInfo->param = i;
       pInfo->paramFreeFp = taosMemoryFree;
       pInfo->requestId = generateRequestId();
       pInfo->requestObjRefId = 0;
@@ -874,6 +883,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
 
   taosThreadMutexLock(&clientHbMgr.lock);
   taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr);
+  pAppHbMgr->idx = taosArrayGetSize(clientHbMgr.appHbMgrs) - 1;
   taosThreadMutexUnlock(&clientHbMgr.lock);
 
   return pAppHbMgr;

source/client/src/clientImpl.c

@@ -1307,6 +1307,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
   tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app));
   tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user));
   tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd));
+  tstrncpy(connectReq.sVer, version, sizeof(connectReq.sVer));
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = taosMemoryMalloc(contLen);

source/client/src/clientMain.c

@@ -271,6 +271,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     SReqResultInfo *pResultInfo;
     if (msg->resIter == -1) {
       pResultInfo = tmqGetNextResInfo(res, true);
+      tscDebug("consumer:0x%" PRIx64 ", vgId:%d, numOfRows:%" PRId64 ", total rows:%" PRId64, msg->rsp.head.consumerId,
+               msg->vgId, pResultInfo->numOfRows, pResultInfo->totalRows);
     } else {
       pResultInfo = tmqGetCurResInfo(res);
     }
@@ -281,7 +283,13 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
       return pResultInfo->row;
     } else {
       pResultInfo = tmqGetNextResInfo(res, true);
-      if (pResultInfo == NULL) return NULL;
+      if (pResultInfo == NULL) {
+        return NULL;
+      }
+
+      tscDebug("consumer:0x%" PRIx64 " vgId:%d, numOfRows:%" PRId64 ", total rows:%" PRId64, msg->rsp.head.consumerId,
+               msg->vgId, pResultInfo->numOfRows, pResultInfo->totalRows);
+
       doSetOneRowPtr(pResultInfo);
       pResultInfo->current += 1;
       return pResultInfo->row;

source/client/src/clientMsgHandler.c

@@ -18,10 +18,12 @@
 #include "clientLog.h"
 #include "os.h"
 #include "query.h"
-#include "tdef.h"
-#include "tname.h"
-#include "tdatablock.h"
 #include "systable.h"
+#include "tdatablock.h"
+#include "tdef.h"
+#include "tglobal.h"
+#include "tname.h"
+#include "tversion.h"
 
 static void setErrno(SRequestObj* pRequest, int32_t code) {
   pRequest->code = code;
@@ -47,11 +49,11 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
 }
 
 int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
-  SRequestObj *pRequest = acquireRequest(*(int64_t*)param);
+  SRequestObj* pRequest = acquireRequest(*(int64_t*)param);
   if (NULL == pRequest) {
     goto End;
   }
-  
+
   if (code != TSDB_CODE_SUCCESS) {
     setErrno(pRequest, code);
     tsem_post(&pRequest->body.rspSem);
@@ -65,7 +67,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
     tsem_post(&pRequest->body.rspSem);
     goto End;
   }
-  
+
   SConnectRsp connectRsp = {0};
   if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) {
     code = TSDB_CODE_TSC_INVALID_VERSION;
@@ -74,6 +76,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
     goto End;
   }
 
+  if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 2)) != 0) {
+    setErrno(pRequest, code);
+    tsem_post(&pRequest->body.rspSem);
+    goto End;
+  }
+
   int32_t now = taosGetTimestampSec();
   int32_t delta = abs(now - connectRsp.svrTimestamp);
   if (delta > timestampDeltaLimit) {
@@ -127,14 +135,14 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
            pTscObj->pAppInfo->numOfConns);
-           
+
   tsem_post(&pRequest->body.rspSem);
 End:
 
   if (pRequest) {
     releaseRequest(pRequest->self);
   }
-  
+
   taosMemoryFree(param);
   taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
@@ -166,18 +174,18 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     struct SCatalog* pCatalog = NULL;
     int32_t          code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
     if (TSDB_CODE_SUCCESS == code) {
-      STscObj*             pTscObj = pRequest->pTscObj;
+      STscObj* pTscObj = pRequest->pTscObj;
 
       SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
                                .requestId = pRequest->requestId,
                                .requestObjRefId = pRequest->self,
                                .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
-      char dbFName[TSDB_DB_FNAME_LEN];
+      char             dbFName[TSDB_DB_FNAME_LEN];
       snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB);
       catalogRefreshDBVgInfo(pCatalog, &conn, dbFName);
       snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB);
       catalogRefreshDBVgInfo(pCatalog, &conn, dbFName);
-    }  
+    }
   }
 
   if (pRequest->body.queryFp) {
@@ -197,7 +205,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp);
     struct SCatalog* pCatalog = NULL;
 
-    if (usedbRsp.vgVersion >= 0) { // cached in local
+    if (usedbRsp.vgVersion >= 0) {  // cached in local
       uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId;
       int32_t  code1 = catalogGetHandle(clusterId, &pCatalog);
       if (code1 != TSDB_CODE_SUCCESS) {
@@ -289,7 +297,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
 }
 
 int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
-  if(pMsg == NULL || param == NULL){
+  if (pMsg == NULL || param == NULL) {
     return TSDB_CODE_TSC_INVALID_INPUT;
   }
   SRequestObj* pRequest = param;
@@ -344,13 +352,13 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     int32_t          code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
     if (TSDB_CODE_SUCCESS == code) {
       catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid);
-      STscObj*             pTscObj = pRequest->pTscObj;
+      STscObj* pTscObj = pRequest->pTscObj;
 
       SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
                                .requestId = pRequest->requestId,
                                .requestObjRefId = pRequest->self,
                                .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
-      char dbFName[TSDB_DB_FNAME_LEN];
+      char             dbFName[TSDB_DB_FNAME_LEN];
       snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB);
       catalogRefreshDBVgInfo(pCatalog, &conn, dbFName);
       snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB);
@@ -474,8 +482,9 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
   int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS);
   blockDataDestroy(pBlock);
 
-  if(len != rspSize - sizeof(SRetrieveTableRsp)){
+  if (len != rspSize - sizeof(SRetrieveTableRsp)) {
-    uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t) (rspSize - sizeof(SRetrieveTableRsp)));
+    uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
+           (uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
     return TSDB_CODE_TSC_INVALID_INPUT;
   }
 

source/client/src/clientSml.c

@@ -1073,8 +1073,6 @@ void smlDestroyInfo(SSmlHandle *info) {
   taosArrayDestroy(info->valueJsonArray);
 
   taosArrayDestroy(info->preLineTagKV);
-  taosArrayDestroy(info->maxTagKVs);
-  taosArrayDestroy(info->preLineColKV);
 
   if (!info->dataFormat) {
     for (int i = 0; i < info->lineNum; i++) {
@@ -1117,8 +1115,6 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
   info->tagJsonArray = taosArrayInit(8, POINTER_BYTES);
   info->valueJsonArray = taosArrayInit(8, POINTER_BYTES);
   info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv));
-  info->maxTagKVs = taosArrayInit(8, sizeof(SSmlKv));
-  info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv));
 
   if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables) {
     uError("create SSmlHandle failed");
@@ -1141,6 +1137,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
   for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
     taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
+    if(terrno == TSDB_CODE_DUP_KEY){return terrno;}
   }
 
   taosArrayPush(colsArray, &kvHash);
@@ -1207,6 +1204,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
 
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
       smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
+      if(terrno == TSDB_CODE_DUP_KEY){return terrno;}
       smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
       taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
     }

source/client/src/clientSmlJson.c

@@ -683,13 +683,10 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
 
   int     cnt = 0;
   SArray *preLineKV = info->preLineTagKV;
-  SArray *maxKVs = info->maxTagKVs;
-  bool    isSuperKVInit = true;
-  SArray *superKV = NULL;
   if (info->dataFormat) {
     if (unlikely(!isSameMeasure)) {
       SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta  *sMeta = NULL;
+      SSmlSTableMeta *sMeta = NULL;
       if (unlikely(tmp == NULL)) {
         STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
         if (pTableMeta == NULL) {
@@ -700,18 +697,16 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
         sMeta = smlBuildSTableMeta(info->dataFormat);
         sMeta->tableMeta = pTableMeta;
         taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
+        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
+          SSchema *tag = pTableMeta->schema + i;
+          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
+          taosArrayPush(sMeta->tags, &kv);
+        }
         tmp = &sMeta;
       }
       info->currSTableMeta = (*tmp)->tableMeta;
-      superKV = (*tmp)->tags;
+      info->maxTagKVs = (*tmp)->tags;
-
-      if (unlikely(taosArrayGetSize(superKV) == 0)) {
-        isSuperKVInit = false;
-      }
-      taosArrayClear(maxKVs);
     }
-  } else {
-    taosArrayClear(maxKVs);
   }
   taosArrayClear(preLineKV);
 
@@ -747,58 +742,21 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
         return TSDB_CODE_SUCCESS;
       }
 
-      if (isSameMeasure) {
+      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        if (unlikely(cnt >= taosArrayGetSize(maxKVs))) {
+        info->dataFormat = false;
-          info->dataFormat = false;
+        info->reRun = true;
-          info->reRun = true;
+        return TSDB_CODE_SUCCESS;
-          return TSDB_CODE_SUCCESS;
+      }
-        }
+      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-        SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt);
+      if (unlikely(!IS_SAME_KEY)) {
-        if (unlikely(kv.length > maxKV->length)) {
+        info->dataFormat = false;
-          maxKV->length = kv.length;
+        info->reRun = true;
-          SSmlSTableMeta **tableMeta =
+        return TSDB_CODE_SUCCESS;
-              (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
+      }
-          if (unlikely(NULL == tableMeta)) {
+      if (unlikely(kv.length > maxKV->length)) {
-            uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
+        maxKV->length = kv.length;
-            return TSDB_CODE_SML_INTERNAL_ERROR;
+        info->needModifySchema = true;
-          }
-
-          SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt);
-          oldKV->length = kv.length;
-          info->needModifySchema = true;
-        }
-        if (unlikely(!IS_SAME_KEY)) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-      } else {
-        if (isSuperKVInit) {
-          if (unlikely(cnt >= taosArrayGetSize(superKV))) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-          SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-          if (unlikely(kv.length > maxKV->length)) {
-            maxKV->length = kv.length;
-          } else {
-            kv.length = maxKV->length;
-          }
-          info->needModifySchema = true;
-
-          if (unlikely(!IS_SAME_KEY)) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-        } else {
-          taosArrayPush(superKV, &kv);
-        }
-        taosArrayPush(maxKVs, &kv);
       }
-    } else {
-      taosArrayPush(maxKVs, &kv);
     }
     taosArrayPush(preLineKV, &kv);
     cnt++;
@@ -1214,7 +1172,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
     return TSDB_CODE_INVALID_TIMESTAMP;
   } else if (elements->timestamp[0] == '{') {
     char tmp = elements->timestamp[elements->timestampLen];
-    elements->cols[elements->timestampLen] = '\0';
+    elements->timestamp[elements->timestampLen] = '\0';
     cJSON *tsJson = cJSON_Parse(elements->timestamp);
     ts = smlParseTSFromJSON(info, tsJson);
     if (unlikely(ts < 0)) {

source/client/src/clientSmlLine.c

@@ -144,13 +144,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
   int     cnt = 0;
   SArray *preLineKV = info->preLineTagKV;
-  SArray *maxKVs = info->maxTagKVs;
-  bool    isSuperKVInit = true;
-  SArray *superKV = NULL;
   if (info->dataFormat) {
     if (unlikely(!isSameMeasure)) {
       SSmlSTableMeta **tmp =
           (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
+
       SSmlSTableMeta *sMeta = NULL;
       if (unlikely(tmp == NULL)) {
         STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
@@ -162,18 +160,16 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         sMeta = smlBuildSTableMeta(info->dataFormat);
         sMeta->tableMeta = pTableMeta;
         taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
+        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
+          SSchema *tag = pTableMeta->schema + i;
+          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
+          taosArrayPush(sMeta->tags, &kv);
+        }
         tmp = &sMeta;
       }
       info->currSTableMeta = (*tmp)->tableMeta;
-      superKV = (*tmp)->tags;
+      info->maxTagKVs = (*tmp)->tags;
-
-      if (unlikely(taosArrayGetSize(superKV) == 0)) {
-        isSuperKVInit = false;
-      }
-      taosArrayClear(maxKVs);
     }
-  } else {
-    taosArrayClear(maxKVs);
   }
   taosArrayClear(preLineKV);
 
@@ -252,58 +248,23 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         return TSDB_CODE_SUCCESS;
       }
 
-      if (isSameMeasure) {
+      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        if (unlikely(cnt >= taosArrayGetSize(maxKVs))) {
+        info->dataFormat = false;
-          info->dataFormat = false;
+        info->reRun = true;
-          info->reRun = true;
+        return TSDB_CODE_SUCCESS;
-          return TSDB_CODE_SUCCESS;
+      }
-        }
+      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-        SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt);
+
-        if (unlikely(kv.length > maxKV->length)) {
+      if (unlikely(!IS_SAME_KEY)) {
-          maxKV->length = kv.length;
+        info->dataFormat = false;
-          SSmlSTableMeta **tableMeta =
+        info->reRun = true;
-              (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
+        return TSDB_CODE_SUCCESS;
-          if (unlikely(NULL == tableMeta)) {
+      }
-            uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
+
-            return TSDB_CODE_SML_INTERNAL_ERROR;
+      if (unlikely(kv.length > maxKV->length)) {
-          }
+        maxKV->length = kv.length;
-
+        info->needModifySchema = true;
-          SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt);
-          oldKV->length = kv.length;
-          info->needModifySchema = true;
-        }
-        if (unlikely(!IS_SAME_KEY)) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-      } else {
-        if (isSuperKVInit) {
-          if (unlikely(cnt >= taosArrayGetSize(superKV))) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-          SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-          if (unlikely(kv.length > maxKV->length)) {
-            maxKV->length = kv.length;
-          } else {
-            kv.length = maxKV->length;
-          }
-          info->needModifySchema = true;
-
-          if (unlikely(!IS_SAME_KEY)) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-        } else {
-          taosArrayPush(superKV, &kv);
-        }
-        taosArrayPush(maxKVs, &kv);
       }
-    } else {
-      taosArrayPush(maxKVs, &kv);
     }
     taosArrayPush(preLineKV, &kv);
 
@@ -344,9 +305,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
                              bool isSameCTable) {
   int     cnt = 0;
-  SArray *preLineKV = info->preLineColKV;
-  bool    isSuperKVInit = true;
-  SArray *superKV = NULL;
   if (info->dataFormat) {
     if (unlikely(!isSameCTable)) {
       SSmlTableInfo **oneTable =
@@ -361,7 +319,6 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     if (unlikely(!isSameMeasure)) {
       SSmlSTableMeta **tmp =
           (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
       if (unlikely(tmp == NULL)) {
         STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
         if (pTableMeta == NULL) {
@@ -369,17 +326,23 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
           info->reRun = true;
           return TSDB_CODE_SUCCESS;
         }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
+        *tmp = smlBuildSTableMeta(info->dataFormat);
-        sMeta->tableMeta = pTableMeta;
+        (*tmp)->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
+        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
-        tmp = &sMeta;
+
+        for(int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++){
+          SSchema *tag = pTableMeta->schema + i;
+          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type };
+          if(tag->type == TSDB_DATA_TYPE_NCHAR){
+            kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+          }else if(tag->type == TSDB_DATA_TYPE_BINARY){
+            kv.length = tag->bytes - VARSTR_HEADER_SIZE;
+          }
+          taosArrayPush((*tmp)->cols, &kv);
+        }
       }
       info->currSTableMeta = (*tmp)->tableMeta;
-      superKV = (*tmp)->cols;
+      info->masColKVs = (*tmp)->cols;
-      if (unlikely(taosArrayGetSize(superKV) == 0)) {
-        isSuperKVInit = false;
-      }
-      taosArrayClear(preLineKV);
     }
   }
 
@@ -478,69 +441,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         info->reRun = true;
         return TSDB_CODE_SUCCESS;
       }
+      if (cnt >= taosArrayGetSize(info->masColKVs)) {
+        info->dataFormat = false;
+        info->reRun = true;
+        return TSDB_CODE_SUCCESS;
+      }
+      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt);
+      if (kv.type != maxKV->type) {
+        info->dataFormat = false;
+        info->reRun = true;
+        return TSDB_CODE_SUCCESS;
+      }
+      if (unlikely(!IS_SAME_KEY)) {
+        info->dataFormat = false;
+        info->reRun = true;
+        return TSDB_CODE_SUCCESS;
+      }
 
-      if (isSameMeasure) {
+      if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) {
-        if (cnt >= taosArrayGetSize(preLineKV)) {
+        maxKV->length = kv.length;
-          info->dataFormat = false;
+        info->needModifySchema = true;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        SSmlKv *maxKV = (SSmlKv *)taosArrayGet(preLineKV, cnt);
-        if (kv.type != maxKV->type) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-
-        if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) {
-          maxKV->length = kv.length;
-          SSmlSTableMeta **tableMeta =
-              (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-          if (unlikely(NULL == tableMeta)) {
-            uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
-            return TSDB_CODE_SML_INTERNAL_ERROR;
-          }
-
-          SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->cols, cnt);
-          oldKV->length = kv.length;
-          info->needModifySchema = true;
-        }
-        if (unlikely(!IS_SAME_KEY)) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-      } else {
-        if (isSuperKVInit) {
-          if (unlikely(cnt >= taosArrayGetSize(superKV))) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-          SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-          if (unlikely(kv.type != maxKV->type)) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-
-          if (IS_VAR_DATA_TYPE(kv.type)) {
-            if (kv.length > maxKV->length) {
-              maxKV->length = kv.length;
-            } else {
-              kv.length = maxKV->length;
-            }
-            info->needModifySchema = true;
-          }
-          if (unlikely(!IS_SAME_KEY)) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-        } else {
-          taosArrayPush(superKV, &kv);
-        }
-        taosArrayPush(preLineKV, &kv);
       }
     } else {
       if (currElement->colArray == NULL) {

source/client/src/clientSmlTelnet.c

@@ -79,13 +79,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
 
   int     cnt = 0;
   SArray *preLineKV = info->preLineTagKV;
-  SArray *maxKVs = info->maxTagKVs;
-  bool    isSuperKVInit = true;
-  SArray *superKV = NULL;
   if (info->dataFormat) {
     if (!isSameMeasure) {
       SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta  *sMeta = NULL;
+      SSmlSTableMeta *sMeta = NULL;
       if (unlikely(tmp == NULL)) {
         STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
         if (pTableMeta == NULL) {
@@ -96,18 +93,16 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
         sMeta = smlBuildSTableMeta(info->dataFormat);
         sMeta->tableMeta = pTableMeta;
         taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
+        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
+          SSchema *tag = pTableMeta->schema + i;
+          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
+          taosArrayPush(sMeta->tags, &kv);
+        }
         tmp = &sMeta;
       }
       info->currSTableMeta = (*tmp)->tableMeta;
-      superKV = (*tmp)->tags;
+      info->maxTagKVs = (*tmp)->tags;
-
-      if (unlikely(taosArrayGetSize(superKV) == 0)) {
-        isSuperKVInit = false;
-      }
-      taosArrayClear(maxKVs);
     }
-  } else {
-    taosArrayClear(maxKVs);
   }
 
   taosArrayClear(preLineKV);
@@ -175,59 +170,21 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
         info->reRun = true;
         return TSDB_CODE_SUCCESS;
       }
-
+      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-      if (isSameMeasure) {
+        info->dataFormat = false;
-        if (unlikely(cnt >= taosArrayGetSize(maxKVs))) {
+        info->reRun = true;
-          info->dataFormat = false;
+        return TSDB_CODE_SUCCESS;
-          info->reRun = true;
+      }
-          return TSDB_CODE_SUCCESS;
+      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-        }
+      if (unlikely(!IS_SAME_KEY)) {
-        SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt);
+        info->dataFormat = false;
-        if (unlikely(kv.length > maxKV->length)) {
+        info->reRun = true;
-          maxKV->length = kv.length;
+        return TSDB_CODE_SUCCESS;
-          SSmlSTableMeta **tableMeta =
+      }
-              (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
+      if (unlikely(kv.length > maxKV->length)) {
-          if (unlikely(NULL == tableMeta)) {
+        maxKV->length = kv.length;
-            uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
+        info->needModifySchema = true;
-            return TSDB_CODE_SML_INTERNAL_ERROR;
-          }
-
-          SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt);
-          oldKV->length = kv.length;
-          info->needModifySchema = true;
-        }
-        if (unlikely(!IS_SAME_KEY)) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-      } else {
-        if (isSuperKVInit) {
-          if (unlikely(cnt >= taosArrayGetSize(superKV))) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-          SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-          if (unlikely(kv.length > maxKV->length)) {
-            maxKV->length = kv.length;
-          } else {
-            kv.length = maxKV->length;
-          }
-          info->needModifySchema = true;
-
-          if (unlikely(!IS_SAME_KEY)) {
-            info->dataFormat = false;
-            info->reRun = true;
-            return TSDB_CODE_SUCCESS;
-          }
-        } else {
-          taosArrayPush(superKV, &kv);
-        }
-        taosArrayPush(maxKVs, &kv);
       }
-    } else {
-      taosArrayPush(maxKVs, &kv);
     }
     taosArrayPush(preLineKV, &kv);
     cnt++;

source/client/src/clientTmq.c

@@ -192,6 +192,7 @@ typedef struct {
   SMqClientTopic* pTopic;
   int32_t         vgId;
   tsem_t          rspSem;
+  uint64_t        requestId; // request id for debug purpose
 } SMqPollCbParam;
 
 typedef struct {
@@ -1054,7 +1055,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
     pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer);
   }
 
-  tscInfo("consumer:0x%" PRIx64 " is setup, consumer groupId %s", pTmq->consumerId, pTmq->groupId);
+  tscInfo("consumer:0x%" PRIx64 " is setup, groupId:%s", pTmq->consumerId, pTmq->groupId);
   return pTmq;
 
 FAIL:
@@ -1075,7 +1076,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   SCMSubscribeReq req = {0};
   int32_t         code = 0;
 
-  tscDebug("consumer:0x%" PRIx64 " subscribe %d topics", tmq->consumerId, sz);
+  tscDebug("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz);
 
   req.consumerId = tmq->consumerId;
   tstrncpy(req.clientId, tmq->clientId, 256);
@@ -1161,7 +1162,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
 
   int32_t retryCnt = 0;
   while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
-    if (retryCnt++ > 10) {
+    if (retryCnt++ > 40) {
       goto FAIL;
     }
 
@@ -1212,30 +1213,39 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     return -1;
   }
 
-  int32_t epoch = pParam->epoch;
+  int32_t  epoch = pParam->epoch;
-  int32_t vgId = pParam->vgId;
+  int32_t  vgId = pParam->vgId;
+  uint64_t requestId = pParam->requestId;
+
   taosMemoryFree(pParam);
+
   if (code != 0) {
-    tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr());
+    tscWarn("consumer:0x%"PRIx64" msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%"PRIx64, tmq->consumerId, vgId,
+        epoch, tstrerror(code), requestId);
+
     if (pMsg->pData) taosMemoryFree(pMsg->pData);
     if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet);
 
+    // in case of consumer mismatch, wait for 500ms and retry
     if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) {
+      taosMsleep(500);
       atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER);
-      goto CREATE_MSG_FAIL;
+      tscDebug("consumer:0x%" PRIx64" wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId);
-    }
+    } else if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
-    if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
       SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0);
       if (pRspWrapper == NULL) {
-        tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch);
+        tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d since out of memory, reqId:0x%" PRIx64,
+                tmq->consumerId, vgId, epoch, requestId);
         goto CREATE_MSG_FAIL;
       }
+
       pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP;
       /*pRspWrapper->vgHandle = pVg;*/
       /*pRspWrapper->topicHandle = pTopic;*/
       taosWriteQitem(tmq->mqueue, pRspWrapper);
       tsem_post(&tmq->rspSem);
     }
+
     goto CREATE_MSG_FAIL;
   }
 
@@ -1243,8 +1253,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
     // do not write into queue since updating epoch reset
-    tscWarn("msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d", vgId, msgEpoch,
+    tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%"PRIx64,
-            tmqEpoch);
+            tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId);
+
     tsem_post(&tmq->rspSem);
     taosMemoryFree(pMsg->pData);
     taosMemoryFree(pMsg->pEpSet);
@@ -1252,7 +1263,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   if (msgEpoch != tmqEpoch) {
-    tscWarn("mismatch rsp from vgId:%d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch);
+    tscWarn("consumer:0x%"PRIx64" mismatch rsp from vgId:%d, epoch %d, current epoch %d, reqId:0x%"PRIx64, tmq->consumerId, vgId,
+        msgEpoch, tmqEpoch, requestId);
   }
 
   // handle meta rsp
@@ -1262,7 +1274,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   if (pRspWrapper == NULL) {
     taosMemoryFree(pMsg->pData);
     taosMemoryFree(pMsg->pEpSet);
-    tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch);
+    tscWarn("consumer:0x%"PRIx64" msg discard from vgId:%d, epoch %d since out of memory", tmq->consumerId, vgId, epoch);
     goto CREATE_MSG_FAIL;
   }
 
@@ -1277,10 +1289,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
 
-    tscDebug("consumer:0x%" PRIx64 ", recv poll: vgId:%d, req offset %" PRId64 ", rsp offset %" PRId64 " type %d",
+    tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req offset:%" PRId64 ", rsp offset:%" PRId64 " type %d, reqId:0x%"PRIx64,
              tmq->consumerId, pVg->vgId, pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version,
-             rspType);
+             rspType, requestId);
-
   } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
     SDecoder decoder;
     tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
@@ -1298,16 +1309,19 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pData);
   taosMemoryFree(pMsg->pEpSet);
 
-  tscDebug("consumer:0x%" PRIx64 ", put poll res into mqueue %p", tmq->consumerId, pRspWrapper);
+  tscDebug("consumer:0x%" PRIx64 ", put poll res into mqueue, total in queue:%d, reqId:0x%" PRIx64, tmq->consumerId,
+           tmq->mqueue->numOfItems, requestId);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
   tsem_post(&tmq->rspSem);
 
   return 0;
+
 CREATE_MSG_FAIL:
   if (epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
+
   tsem_post(&tmq->rspSem);
   return -1;
 }
@@ -1344,7 +1358,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
         sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId);
         char buf[80];
         tFormatOffset(buf, 80, &pVgCur->currentOffset);
-        tscDebug("consumer:0x%" PRIx64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch,
+        tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch,
                  pVgCur->vgId, vgKey, buf);
         taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(STqOffsetVal));
       }
@@ -1552,7 +1566,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) {
       .handle = NULL,
   };
 
-  sendInfo->requestId = tmq->consumerId;
+  sendInfo->requestId = generateRequestId();
   sendInfo->requestObjRefId = 0;
   sendInfo->param = pParam;
   sendInfo->fp = tmqAskEpCb;
@@ -1560,7 +1574,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) {
 
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
   tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%" PRIx64, tmq->consumerId, async,
-           tmq->consumerId);
+           sendInfo->requestId);
 
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
@@ -1674,6 +1688,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
   pParam->pVg = pVg;  // pVg may be released,fix it
   pParam->pTopic = pTopic;
   pParam->vgId = pVg->vgId;
+  pParam->requestId = req.reqId;
 
   SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
   if (sendInfo == NULL) {
@@ -1698,7 +1713,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
   char    offsetFormatBuf[80];
   tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->currentOffset);
 
-  tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:0x%" PRIx64,
+  tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
            pTmq->consumerId, pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId);
   asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
 
@@ -1722,10 +1737,9 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus == TMQ_VG_STATUS__WAIT) {
         int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
-        tscDebug("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch,
+        tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch,
                  pVg->vgId, vgSkipCnt);
         continue;
-        /*if (vgSkipCnt < 10000) continue;*/
 #if 0
         if (skipCnt < 30000) {
           continue;
@@ -1767,7 +1781,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
 }
 
 void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
-  tscDebug("consumer:0x%" PRIx64 " start to handle the rsp", tmq->consumerId);
+  tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems);
 
   while (1) {
     SMqRspWrapper* rspWrapper = NULL;
@@ -1785,25 +1799,32 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
     if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) {
       taosFreeQitem(rspWrapper);
       terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
+      tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(terrno));
       return NULL;
     } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
-      tscDebug("consumer:0x%" PRIx64 " process poll rsp", tmq->consumerId);
+
       /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
       int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
       if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
         SMqClientVg* pVg = pollRspWrapper->vgHandle;
-        /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset,
-         * rspMsg->msg.rspOffset);*/
         pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+
         if (pollRspWrapper->dataRsp.blockNum == 0) {
+          tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d", tmq->consumerId, pVg->vgId);
           taosFreeQitem(pollRspWrapper);
           rspWrapper = NULL;
           continue;
         }
+
         // build rsp
+        char buf[80];
+        tFormatOffset(buf, 80, &pVg->currentOffset);
         SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
+        tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d", tmq->consumerId,
+                     pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum);
+
         taosFreeQitem(pollRspWrapper);
         return pRsp;
       } else {
@@ -1876,6 +1897,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
       }
     }
   }
+
+  tscDebug("consumer:0x%" PRIx64 " handle the rsp completed", tmq->consumerId);
 }
 
 TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
@@ -1896,17 +1919,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
   // in no topic status, delayed task also need to be processed
   if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) {
     tscDebug("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId);
+    taosMsleep(500);  //     sleep for a while
     return NULL;
   }
 
   if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) {
     int32_t retryCnt = 0;
     while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
-      if (retryCnt++ > 10) {
+      if (retryCnt++ > 40) {
         return NULL;
       }
 
-      tscDebug("consumer:0x%" PRIx64 " not ready, retry:%d/10 in 500ms", tmq->consumerId, retryCnt);
+      tscDebug("consumer:0x%" PRIx64 " not ready, retry:%d/40 in 500ms", tmq->consumerId, retryCnt);
       taosMsleep(500);
     }
   }

source/client/test/clientTests.cpp

@@ -925,7 +925,7 @@ TEST(clientCase, subscription_test) {
 
   // 创建订阅 topics 列表
   tmq_list_t* topicList = tmq_list_new();
-  tmq_list_append(topicList, "topic_t1");
+//  tmq_list_append(topicList, "topic_t1");
 
   // 启动订阅
   tmq_subscribe(tmq, topicList);
@@ -938,6 +938,8 @@ TEST(clientCase, subscription_test) {
   int32_t     msgCnt = 0;
   int32_t     timeout = 5000;
 
+  int32_t count = 0;
+
   while (1) {
     TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
     if (pRes) {
@@ -952,6 +954,11 @@ TEST(clientCase, subscription_test) {
       printf("db: %s\n", dbName);
       printf("vgroup id: %d\n", vgroupId);
 
+      if (count ++ > 20) {
+        tmq_unsubscribe(tmq);
+        break;
+      }
+
       while (1) {
         TAOS_ROW row = taos_fetch_row(pRes);
         if (row == NULL) break;

source/common/src/systable.c

@@ -33,6 +33,7 @@ static const SSysDbTableSchema dnodesSchema[] = {
     {.name = "support_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
+    {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},   
     {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
@@ -42,6 +43,7 @@ static const SSysDbTableSchema mnodesSchema[] = {
     {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
+    {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
 };
 
 static const SSysDbTableSchema modulesSchema[] = {

source/common/src/tdatablock.c

@@ -2003,7 +2003,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
       switch (pColInfoData->info.type) {
         case TSDB_DATA_TYPE_TIMESTAMP:
           memset(pBuf, 0, sizeof(pBuf));
-          formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
+          formatTimestamp(pBuf, *(uint64_t*)var, pColInfoData->info.precision);
           len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf);
           if (len >= size - 1) return dumpBuf;
           break;

source/common/src/tglobal.c

@@ -41,8 +41,8 @@ bool    tsPrintAuth = false;
 
 // queue & threads
 int32_t tsNumOfRpcThreads = 1;
-int32_t tsNumOfRpcSessions = 5000;
+int32_t tsNumOfRpcSessions = 10000;
-int32_t tsTimeToGetAvailableConn = 100000;
+int32_t tsTimeToGetAvailableConn = 500000;
 int32_t tsNumOfCommitThreads = 2;
 int32_t tsNumOfTaskQueueThreads = 4;
 int32_t tsNumOfMnodeQueryThreads = 4;
@@ -56,6 +56,7 @@ int32_t tsNumOfQnodeQueryThreads = 4;
 int32_t tsNumOfQnodeFetchThreads = 1;
 int32_t tsNumOfSnodeStreamThreads = 4;
 int32_t tsNumOfSnodeWriteThreads = 1;
+
 // sync raft
 int32_t tsElectInterval = 25 * 1000;
 int32_t tsHeartbeatInterval = 1000;
@@ -64,6 +65,10 @@ int32_t tsHeartbeatTimeout = 20 * 1000;
 // vnode
 int64_t tsVndCommitMaxIntervalMs = 600 * 1000;
 
+// mnode
+int64_t tsMndSdbWriteDelta = 200;
+int64_t tsMndLogRetention = 2000;
+
 // monitor
 bool     tsEnableMonitor = true;
 int32_t  tsMonitorInterval = 30;
@@ -327,6 +332,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1;
   if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1;
 
+  tsNumOfRpcThreads = tsNumOfCores / 2;
+  tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
+  if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1;
+
   tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000);
   if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, 0) != 0) return -1;
 
@@ -456,6 +465,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, 0) != 0) return -1;
 
+  if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, 0) != 0) return -1;
+  if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, 0) != 0) return -1;
+
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1;
   if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1;
@@ -521,7 +533,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
 
   pItem = cfgGetItem(tsCfg, "numOfRpcSessions");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
-    tsNumOfRpcSessions = 2000;
     tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000);
     pItem->i32 = tsNumOfRpcSessions;
     pItem->stype = stype;
@@ -529,7 +540,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
 
   pItem = cfgGetItem(tsCfg, "timeToGetAvailableConn");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
-    tsTimeToGetAvailableConn = 1000;
     tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000);
     pItem->i32 = tsTimeToGetAvailableConn;
     pItem->stype = stype;
@@ -717,6 +727,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
 
   tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
 
+  tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
   tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;
 
   tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
@@ -803,6 +814,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsVndCommitMaxIntervalMs = cfgGetItem(pCfg, "vndCommitMaxInterval")->i64;
 
+  tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64;
+  tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64;
+
   tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;
   tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs));
   tstrncpy(tsUdfdLdLibPath, cfgGetItem(pCfg, "udfdLdLibPath")->str, sizeof(tsUdfdLdLibPath));
@@ -1240,6 +1254,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
   taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false);
 
   if (taosMulModeMkDir(tsLogDir, 0777) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
     uError("failed to create dir:%s since %s", tsLogDir, terrstr());
     cfgCleanup(pCfg);
     return -1;

source/common/src/tmsg.c

@@ -2531,6 +2531,8 @@ int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq)
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->timeRange.skey) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->timeRange.ekey) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -2544,6 +2546,8 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->timeRange.skey) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->timeRange.ekey) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -2865,12 +2869,19 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
   if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->walFsyncPeriod) < 0) return -1;
+  if (tEncodeI16(&encoder, pRsp->hashPrefix) < 0) return -1;
+  if (tEncodeI16(&encoder, pRsp->hashSuffix) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1;
   if (tEncodeI8(&encoder, pRsp->cacheLast) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->tsdbPageSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->walRetentionPeriod) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->walRollPeriod) < 0) return -1;
+  if (tEncodeI64(&encoder, pRsp->walRetentionSize) < 0) return -1;
+  if (tEncodeI64(&encoder, pRsp->walSegmentSize) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1;
   for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) {
     SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i);
@@ -2905,12 +2916,19 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) {
   if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->walFsyncPeriod) < 0) return -1;
+  if (tDecodeI16(&decoder, &pRsp->hashPrefix) < 0) return -1;
+  if (tDecodeI16(&decoder, &pRsp->hashSuffix) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1;
   if (tDecodeI8(&decoder, &pRsp->cacheLast) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->tsdbPageSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->walRetentionPeriod) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->walRollPeriod) < 0) return -1;
+  if (tDecodeI64(&decoder, &pRsp->walRetentionSize) < 0) return -1;
+  if (tDecodeI64(&decoder, &pRsp->walSegmentSize) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1;
   if (pRsp->numOfRetensions > 0) {
     pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention));
@@ -3723,6 +3741,7 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
   if (tEncodeCStrWithLen(&encoder, pReq->passwd, TSDB_PASSWORD_LEN) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->sVer) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -3742,6 +3761,12 @@ int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->passwd) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->startTime) < 0) return -1;
+  // Check the client version from version 3.0.3.0
+  if (tDecodeIsEnd(&decoder)) {
+    tDecoderClear(&decoder);
+    return TSDB_CODE_VERSION_NOT_COMPATIBLE;
+  }
+  if (tDecodeCStrTo(&decoder, pReq->sVer) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -4090,6 +4115,11 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
   if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->compactStartTime) < 0) return -1;
+
+  // 1.1 add tw.skey and tw.ekey
+  if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1;
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -4102,11 +4132,21 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
   tDecoderInit(&decoder, buf, bufLen);
 
   if (tStartDecode(&decoder) < 0) return -1;
+
   if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->compactStartTime) < 0) return -1;
-  tEndDecode(&decoder);
 
+  // 1.1
+  if (tDecodeIsEnd(&decoder)) {
+    pReq->tw.skey = TSKEY_MIN;
+    pReq->tw.ekey = TSKEY_MAX;
+  } else {
+    if (tDecodeI64(&decoder, &pReq->tw.skey) < 0) return -1;
+    if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1;
+  }
+
+  tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
 }
@@ -6570,7 +6610,7 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
   } else if (pVal->type == TMQ_OFFSET__LOG) {
     snprintf(buf, maxLen, "offset(log) ver:%" PRId64, pVal->version);
   } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
-    snprintf(buf, maxLen, "offset(ss data) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts);
+    snprintf(buf, maxLen, "offset(snapshot) uid:%" PRId64 " ts:%" PRId64, pVal->uid, pVal->ts);
   } else {
     ASSERT(0);
   }

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -61,6 +61,16 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
   rpcFreeCont(pRsp->pCont);
 }
 
+void dmEpSetToStr(char *buf, int32_t len, SEpSet *epSet) {
+  int32_t n = 0;
+  n += snprintf(buf + n, len - n, "%s", "{");
+  for (int i = 0; i < epSet->numOfEps; i++) {
+    n += snprintf(buf + n, len - n, "%s:%d%s", epSet->eps[i].fqdn, epSet->eps[i].port,
+                  (i + 1 < epSet->numOfEps ? ", " : ""));
+  }
+  n += snprintf(buf + n, len - n, "%s", "}");
+}
+
 void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   SStatusReq req = {0};
 
@@ -119,11 +129,10 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   dmGetMnodeEpSet(pMgmt->pData, &epSet);
   rpcSendRecv(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp);
   if (rpcRsp.code != 0) {
-    dError("failed to send status req since %s, numOfEps:%d inUse:%d", tstrerror(rpcRsp.code), epSet.numOfEps,
+    dmRotateMnodeEpSet(pMgmt->pData);
-           epSet.inUse);
+    char tbuf[256];
-    for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+    dmEpSetToStr(tbuf, sizeof(tbuf), &epSet);
-      dDebug("index:%d, mnode ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
+    dError("failed to send status req since %s, epSet:%s, inUse:%d", tstrerror(rpcRsp.code), tbuf, epSet.inUse);
-    }
   }
   dmProcessStatusRsp(pMgmt, &rpcRsp);
 }

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -20,6 +20,8 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) {
   pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad));
   if (pInfo->pVloads == NULL) return;
 
+  tfsUpdateSize(pMgmt->pTfs);
+  
   taosThreadRwlockRdlock(&pMgmt->lock);
 
   void *pIter = taosHashIterate(pMgmt->hash, NULL);

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -192,8 +192,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
       taosWriteQitem(pVnode->pFetchQ, pMsg);
       break;
     case WRITE_QUEUE:
-      if (!osDataSpaceAvailable()) {
+      if (!osDataSpaceSufficient()) {
-        terrno = TSDB_CODE_NO_DISKSPACE;
+        terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE;
         code = terrno;
         dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
         break;

source/dnode/mgmt/node_util/inc/dmUtil.h

@@ -166,6 +166,7 @@ int32_t dmReadEps(SDnodeData *pData);
 int32_t dmWriteEps(SDnodeData *pData);
 void    dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps);
 void    dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
+void    dmRotateMnodeEpSet(SDnodeData *pData);
 void    dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet);
 void    dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
 bool    dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port);

source/dnode/mgmt/node_util/src/dmEps.c

@@ -325,6 +325,28 @@ void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) {
   taosThreadRwlockUnlock(&pData->lock);
 }
 
+static FORCE_INLINE void dmSwapEps(SEp *epLhs, SEp *epRhs) {
+  SEp epTmp;
+
+  epTmp.port = epLhs->port;
+  tstrncpy(epTmp.fqdn, epLhs->fqdn, tListLen(epTmp.fqdn));
+
+  epLhs->port = epRhs->port;
+  tstrncpy(epLhs->fqdn, epRhs->fqdn, tListLen(epLhs->fqdn));
+
+  epRhs->port = epTmp.port;
+  tstrncpy(epRhs->fqdn, epTmp.fqdn, tListLen(epRhs->fqdn));
+}
+
+void dmRotateMnodeEpSet(SDnodeData *pData) {
+  taosThreadRwlockRdlock(&pData->lock);
+  SEpSet *pEpSet = &pData->mnodeEps;
+  for (int i = 1; i < pEpSet->numOfEps; i++) {
+    dmSwapEps(&pEpSet->eps[i - 1], &pEpSet->eps[i]);
+  }
+  taosThreadRwlockUnlock(&pData->lock);
+}
+
 void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) {
   dmGetMnodeEpSet(pData, pEpSet);
   dTrace("msg is redirected, handle:%p num:%d use:%d", pMsg->info.handle, pEpSet->numOfEps, pEpSet->inUse);

source/dnode/mnode/impl/CMakeLists.txt

@@ -2,8 +2,9 @@ aux_source_directory(src MNODE_SRC)
 IF (TD_PRIVILEGE)
   ADD_DEFINITIONS(-D_PRIVILEGE)
 ENDIF ()
-IF (TD_PRIVILEGE)
+IF (TD_ENTERPRISE)
   LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/privilege/src/privilege.c)
+  LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c)
 ENDIF ()
 
 add_library(mnode STATIC ${MNODE_SRC})

source/dnode/mnode/impl/inc/mndDb.h

@@ -33,6 +33,8 @@ bool    mndIsDbReady(SMnode *pMnode, SDbObj *pDb);
 SSdbRaw    *mndDbActionEncode(SDbObj *pDb);
 const char *mndGetDbStr(const char *src);
 
+int32_t mndProcessCompactDbReq(SRpcMsg *pReq);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndDef.h

@@ -108,7 +108,8 @@ typedef enum {
   TRN_STAGE_UNDO_ACTION = 3,
   TRN_STAGE_COMMIT = 4,
   TRN_STAGE_COMMIT_ACTION = 5,
-  TRN_STAGE_FINISHED = 6
+  TRN_STAGE_FINISHED = 6,
+  TRN_STAGE_PRE_FINISH = 7
 } ETrnStage;
 
 typedef enum {
@@ -542,7 +543,7 @@ void*           tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer
 
 typedef struct {
   int32_t vgId;
-  char*   qmsg;
+  char*   qmsg;   //SubPlanToString
   SEpSet  epSet;
 } SMqVgEp;
 

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -43,7 +43,8 @@ int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVg
 int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force);
 int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
                                   SArray *pArray);
-int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs);
+int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs,
+                                    STimeWindow tw);
 
 void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -26,18 +26,18 @@
 #define MND_CONSUMER_VER_NUMBER   1
 #define MND_CONSUMER_RESERVE_SIZE 64
 
-#define MND_CONSUMER_LOST_HB_CNT          3
+#define MND_CONSUMER_LOST_HB_CNT          6
 #define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200
 
-static int8_t mqRebInExecCnt = 0;
+static int32_t mqRebInExecCnt = 0;
 
 static const char *mndConsumerStatusName(int status);
 
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
-static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
+static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer);
 static int32_t mndProcessConsumerMetaMsg(SRpcMsg *pMsg);
-static int32_t mndRetrieveConsumer(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
+static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConsumer(SMnode *pMnode, void *pIter);
 
 static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg);
@@ -76,15 +76,36 @@ int32_t mndInitConsumer(SMnode *pMnode) {
 void mndCleanupConsumer(SMnode *pMnode) {}
 
 bool mndRebTryStart() {
-  int8_t old = atomic_val_compare_exchange_8(&mqRebInExecCnt, 0, 1);
+  int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
+  mInfo("tq timer, rebalance counter old val:%d", old);
   return old == 0;
 }
 
-void mndRebEnd() { atomic_sub_fetch_8(&mqRebInExecCnt, 1); }
+void mndRebEnd() {
+  mndRebCntDec();
+}
 
-void mndRebCntInc() { atomic_add_fetch_8(&mqRebInExecCnt, 1); }
+void mndRebCntInc() {
+  int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1);
+  mInfo("rebalance trans start, rebalance counter:%d", val);
+}
 
-void mndRebCntDec() { atomic_sub_fetch_8(&mqRebInExecCnt, 1); }
+void mndRebCntDec() {
+  while (1) {
+    int32_t val = atomic_load_32(&mqRebInExecCnt);
+    if (val <= 0) {
+      mError("rebalance trans end, rebalance counter:%d should not be less equalled than 0, ignore counter desc", val);
+      break;
+    }
+
+    int32_t newVal = val - 1;
+    int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal);
+    if (oldVal == val) {
+      mInfo("rebalance trans end, rebalance counter:%d", newVal);
+      break;
+    }
+  }
+}
 
 static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
   SMnode             *pMnode = pMsg->info.node;
@@ -298,6 +319,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
       taosRUnLockLatch(&pConsumer->lock);
     } else if (status == MQ_CONSUMER_STATUS__MODIFY) {
       taosRLockLatch(&pConsumer->lock);
+
       int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
       for (int32_t i = 0; i < newTopicNum; i++) {
         char  key[TSDB_SUBSCRIBE_KEY_LEN];
@@ -334,7 +356,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
   } else {
     taosHashCleanup(pRebMsg->rebSubHash);
     rpcFreeCont(pRebMsg);
-    mTrace("mq rebalance finished, no modification");
+    mInfo("mq rebalance finished, no modification");
     mndRebEnd();
   }
   return 0;
@@ -611,10 +633,11 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
 
   pExistedConsumer = mndAcquireConsumer(pMnode, consumerId);
   if (pExistedConsumer == NULL) {
-    mInfo("receive subscribe request from new consumer:0x%" PRIx64" cgroup:%s", consumerId, subscribe.cgroup);
+    mInfo("receive subscribe request from new consumer:0x%" PRIx64" cgroup:%s, numOfTopics:%d", consumerId,
+        subscribe.cgroup, (int32_t) taosArrayGetSize(pTopicList));
 
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
-    tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256);
+    tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId));
 
     // set the update type
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
@@ -943,8 +966,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     pOldConsumer->rebalanceTime = pNewConsumer->upTime;
 
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
-    mDebug("consumer:0x%" PRIx64 " state %s -> %s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
+    mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
-           pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status),
+           pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
+           mndConsumerStatusName(pOldConsumer->status),
            pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
     /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/
@@ -1002,8 +1026,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     pOldConsumer->rebalanceTime = pNewConsumer->upTime;
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
 
-    mDebug("consumer:0x%" PRIx64 " state %s -> %s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
+    mDebug("consumer:0x%" PRIx64 " state %d(%s) -> %d(%s), new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
-           pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status),
+           pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status,
+           mndConsumerStatusName(pOldConsumer->status),
            pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
   }
 
@@ -1044,7 +1069,6 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
     }
 
     taosRLockLatch(&pConsumer->lock);
-
     mDebug("showing consumer:0x%" PRIx64, pConsumer->consumerId);
 
     int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics);

source/dnode/mnode/impl/src/mndDb.c

@@ -41,12 +41,15 @@ static int32_t  mndProcessCreateDbReq(SRpcMsg *pReq);
 static int32_t  mndProcessAlterDbReq(SRpcMsg *pReq);
 static int32_t  mndProcessDropDbReq(SRpcMsg *pReq);
 static int32_t  mndProcessUseDbReq(SRpcMsg *pReq);
-static int32_t  mndProcessCompactDbReq(SRpcMsg *pReq);
 static int32_t  mndProcessTrimDbReq(SRpcMsg *pReq);
 static int32_t  mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity);
 static void     mndCancelGetNextDb(SMnode *pMnode, void *pIter);
 static int32_t  mndProcessGetDbCfgReq(SRpcMsg *pReq);
 
+#ifndef TD_ENTERPRISE
+int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { return TSDB_CODE_OPS_NOT_SUPPORT; }
+#endif
+
 int32_t mndInitDb(SMnode *pMnode) {
   SSdbTable table = {
       .sdbType = SDB_DB,
@@ -884,12 +887,19 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
   cfgRsp.minRows = pDb->cfg.minRows;
   cfgRsp.maxRows = pDb->cfg.maxRows;
   cfgRsp.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
+  cfgRsp.hashPrefix = pDb->cfg.hashPrefix;
+  cfgRsp.hashSuffix = pDb->cfg.hashSuffix;
   cfgRsp.walLevel = pDb->cfg.walLevel;
   cfgRsp.precision = pDb->cfg.precision;
   cfgRsp.compression = pDb->cfg.compression;
   cfgRsp.replications = pDb->cfg.replications;
   cfgRsp.strict = pDb->cfg.strict;
   cfgRsp.cacheLast = pDb->cfg.cacheLast;
+  cfgRsp.tsdbPageSize = pDb->cfg.tsdbPageSize;
+  cfgRsp.walRetentionPeriod = pDb->cfg.walRetentionPeriod;
+  cfgRsp.walRollPeriod = pDb->cfg.walRollPeriod;
+  cfgRsp.walRetentionSize = pDb->cfg.walRetentionSize;
+  cfgRsp.walSegmentSize = pDb->cfg.walSegmentSize;
   cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
   cfgRsp.pRetensions = pDb->cfg.pRetensions;
   cfgRsp.schemaless = pDb->cfg.schemaless;
@@ -1395,98 +1405,6 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
   return 0;
 }
 
-static int32_t mndSetCompactDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) {
-  SDbObj dbObj = {0};
-  memcpy(&dbObj, pDb, sizeof(SDbObj));
-  dbObj.compactStartTime = compactTs;
-
-  SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj);
-  if (pCommitRaw == NULL) return -1;
-  if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
-    sdbFreeRaw(pCommitRaw);
-    return -1;
-  }
-
-  (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
-  return 0;
-}
-
-static int32_t mndSetCompactDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) {
-  SSdb *pSdb = pMnode->pSdb;
-  void *pIter = NULL;
-
-  while (1) {
-    SVgObj *pVgroup = NULL;
-    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
-    if (pIter == NULL) break;
-
-    if (mndVgroupInDb(pVgroup, pDb->uid)) {
-      if (mndBuildCompactVgroupAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) {
-        sdbCancelFetch(pSdb, pIter);
-        sdbRelease(pSdb, pVgroup);
-        return -1;
-      }
-    }
-
-    sdbRelease(pSdb, pVgroup);
-  }
-
-  return 0;
-}
-
-static int32_t mndCompactDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
-  int64_t compactTs = taosGetTimestampMs();
-  int32_t code = -1;
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "compact-db");
-  if (pTrans == NULL) goto _OVER;
-
-  mInfo("trans:%d, used to compact db:%s", pTrans->id, pDb->name);
-  mndTransSetDbName(pTrans, pDb->name, NULL);
-  if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
-  if (mndSetCompactDbCommitLogs(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER;
-  if (mndSetCompactDbRedoActions(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER;
-  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
-  code = 0;
-
-_OVER:
-  mndTransDrop(pTrans);
-  return code;
-}
-
-static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) {
-  SMnode       *pMnode = pReq->info.node;
-  int32_t       code = -1;
-  SDbObj       *pDb = NULL;
-  SCompactDbReq compactReq = {0};
-
-  if (tDeserializeSCompactDbReq(pReq->pCont, pReq->contLen, &compactReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    goto _OVER;
-  }
-
-  mInfo("db:%s, start to compact", compactReq.db);
-
-  pDb = mndAcquireDb(pMnode, compactReq.db);
-  if (pDb == NULL) {
-    goto _OVER;
-  }
-
-  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) {
-    goto _OVER;
-  }
-
-  code = mndCompactDb(pMnode, pReq, pDb);
-  if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
-
-_OVER:
-  if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
-    mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr());
-  }
-
-  mndReleaseDb(pMnode, pDb);
-  return code;
-}
-
 static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) {
   SSdb       *pSdb = pMnode->pSdb;
   SVgObj     *pVgroup = NULL;

source/dnode/mnode/impl/src/mndDnode.c

@@ -1074,6 +1074,9 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false);
 
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->rebootTime, false);
+
     char *b = taosMemoryCalloc(VARSTR_HEADER_SIZE + strlen(offlineReason[pDnode->offlineReason]) + 1, 1);
     STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]);
 

source/dnode/mnode/impl/src/mndMain.c

@@ -344,8 +344,8 @@ static int32_t mndInitWal(SMnode *pMnode) {
       .fsyncPeriod = 0,
       .rollPeriod = -1,
       .segSize = -1,
-      .retentionPeriod = -1,
+      .retentionPeriod = 0,
-      .retentionSize = -1,
+      .retentionSize = 0,
       .level = TAOS_WAL_FSYNC,
   };
 
@@ -370,7 +370,6 @@ static int32_t mndInitSdb(SMnode *pMnode) {
   opt.path = pMnode->path;
   opt.pMnode = pMnode;
   opt.pWal = pMnode->pWal;
-  opt.sync = pMnode->syncMgmt.sync;
 
   pMnode->pSdb = sdbInit(&opt);
   if (pMnode->pSdb == NULL) {
@@ -552,16 +551,7 @@ void mndPreClose(SMnode *pMnode) {
   if (pMnode != NULL) {
     syncLeaderTransfer(pMnode->syncMgmt.sync);
     syncPreStop(pMnode->syncMgmt.sync);
-#if 0
+    sdbWriteFile(pMnode->pSdb, 0);
-    while (syncSnapshotRecving(pMnode->syncMgmt.sync)) {
-      mInfo("vgId:1, snapshot is recving");
-      taosMsleep(300);
-    }
-    while (syncSnapshotSending(pMnode->syncMgmt.sync)) {
-      mInfo("vgId:1, snapshot is sending");
-      taosMsleep(300);
-    }
-#endif
   }
 }
 
@@ -716,6 +706,9 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
   } else if (code == 0) {
     mGTrace("msg:%p, successfully processed", pMsg);
   } else {
+    if (code == -1) {
+      code = terrno;
+    }
     mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, tstrerror(code), pMsg->info.ahandle,
             TMSG_INFO(pMsg->msgType));
   }

source/dnode/mnode/impl/src/mndMnode.c

@@ -644,6 +644,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
 
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->stateStartTime, false);
+
     numOfRows++;
     sdbRelease(pSdb, pObj);
   }

source/dnode/mnode/impl/src/mndProfile.c

@@ -24,7 +24,7 @@
 #include "mndStb.h"
 #include "mndUser.h"
 #include "tglobal.h"
-#include "version.h"
+#include "tversion.h"
 
 typedef struct {
   uint32_t id;
@@ -221,11 +221,18 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
   char            ip[24] = {0};
   const STraceId *trace = &pReq->info.traceId;
 
-  if (tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq) != 0) {
+  if ((code = tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq)) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
+    terrno = (-1 == code ? TSDB_CODE_INVALID_MSG : code);
     goto _OVER;
   }
 
+  if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 2)) != 0) {
+    terrno = code;
+    goto _OVER;
+  }
+
+  code = -1;
+
   taosIp2String(pReq->info.conn.clientIp, ip);
   if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) {
     mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr());

source/dnode/mnode/impl/src/mndStream.c

@@ -1223,7 +1223,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
 
         // node id
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        int32_t nodeId = TMAX(pTask->nodeId, 0);
+        int64_t nodeId = TMAX(pTask->nodeId, 0);
         colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false);
 
         // level

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -224,7 +224,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
             .pVgEp = pVgEp,
         };
         taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-        mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId);
+        mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, sub, pVgEp->vgId, consumerId);
       }
       taosArrayDestroy(pConsumerEp->vgs);
       taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
@@ -263,7 +263,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     imbConsumerNum = totalVgNum % afterRebConsumerNum;
   }
 
-  mInfo("sub:%s mq re-balance %d consumers: at least %d vg each, %d consumer has more vg", sub,
+  mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has more vgs", sub,
         afterRebConsumerNum, minVgCnt, imbConsumerNum);
 
   // 4. first scan: remove consumer more than wanted, put to remove hash
@@ -296,7 +296,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
                 .pVgEp = pVgEp,
             };
             taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-            mInfo("sub:%s mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
+            mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId,
                   pConsumerEp->consumerId);
           }
           imbCnt++;
@@ -311,7 +311,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
               .pVgEp = pVgEp,
           };
           taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-          mInfo("sub:%s mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
+          mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId,
                 pConsumerEp->consumerId);
         }
       }
@@ -329,7 +329,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
       taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
       taosArrayPush(pOutput->newConsumers, &consumerId);
-      mInfo("sub:%s mq rebalance add new consumer:%" PRId64, sub, consumerId);
+      mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, sub, consumerId);
     }
   }
 
@@ -357,7 +357,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 " (second scan) (not enough)", pRebVg->pVgEp->vgId,
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId,
             pConsumerEp->consumerId);
     }
   }
@@ -387,12 +387,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
       if (pRebVg->newConsumerId == pRebVg->oldConsumerId) {
-        mInfo("mq rebalance: skip vg %d for same consumer:%" PRId64 " (second scan)", pRebVg->pVgEp->vgId,
+        mInfo("mq rebalance: skip vg %d for same consumer:%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId,
               pConsumerEp->consumerId);
         continue;
       }
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId,
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId,
             pConsumerEp->consumerId);
     }
   } else {
@@ -427,10 +427,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       if (pIter == NULL) break;
       SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
       int32_t        sz = taosArrayGetSize(pConsumerEp->vgs);
-      mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz);
+      mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", sub, pConsumerEp->consumerId, sz);
       for (int32_t i = 0; i < sz; i++) {
         SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
-        mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRId64, sub, pVgEp->vgId,
+        mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, sub, pVgEp->vgId,
               pConsumerEp->consumerId);
       }
     }
@@ -444,7 +444,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
 
 static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
-  if (pTrans == NULL) return -1;
+  if (pTrans == NULL) {
+    return -1;
+  }
 
   mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
   if (mndTrancCheckConflict(pMnode, pTrans) != 0) {
@@ -591,13 +593,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
       rebOutput.pSub = mndCreateSubscription(pMnode, pTopic, pRebInfo->key);
 
       if (rebOutput.pSub == NULL) {
-        mError("mq rebalance %s failed create sub since %s, abort", pRebInfo->key, terrstr());
+        mError("mq rebalance %s failed create sub since %s, ignore", pRebInfo->key, terrstr());
         taosRUnLockLatch(&pTopic->lock);
         mndReleaseTopic(pMnode, pTopic);
         continue;
       }
-      memcpy(rebOutput.pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN);
 
+      memcpy(rebOutput.pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN);
       taosRUnLockLatch(&pTopic->lock);
       mndReleaseTopic(pMnode, pTopic);
 
@@ -616,9 +618,9 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
 
     // if add more consumer to balanced subscribe,
     // possibly no vg is changed
-
+    // when each topic is re-balanced, issue an trans to save the results in sdb.
     if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) {
-      mError("mq re-balance persist re-balance output error, possibly vnode splitted or dropped");
+      mError("mq re-balance persist output error, possibly vnode splitted or dropped");
     }
 
     taosArrayDestroy(pRebInfo->lostConsumers);
@@ -770,7 +772,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
   if (pSub->unassignedVgs != NULL) {
     int32_t size = (int32_t)taosArrayGetSize(pSub->unassignedVgs);
     for (int32_t i = 0; i < size; ++i) {
-      SMqVgEp *pMqVgEp = taosArrayGet(pSub->unassignedVgs, i);
+      SMqVgEp *pMqVgEp = (SMqVgEp *)taosArrayGetP(pSub->unassignedVgs, i);
       tmsgUpdateDnodeEpSet(&pMqVgEp->epSet);
     }
   }
@@ -780,7 +782,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
       SMqConsumerEp *pConsumerEp = pIter;
       int32_t        size = (int32_t)taosArrayGetSize(pConsumerEp->vgs);
       for (int32_t i = 0; i < size; ++i) {
-        SMqVgEp *pMqVgEp = taosArrayGet(pConsumerEp->vgs, i);
+        SMqVgEp *pMqVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i);
         tmsgUpdateDnodeEpSet(&pMqVgEp->epSet);
       }
       pIter = taosHashIterate(pSub->consumerHash, pIter);
@@ -1017,7 +1019,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
         colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false);
 
-        mDebug("mnd show subscriptions: topic %s, consumer %" PRId64 " cgroup %s vgid %d", varDataVal(topic),
+        mDebug("mnd show subscriptions: topic %s, consumer:%" PRIx64 " cgroup %s vgid %d", varDataVal(topic),
                pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId);
 
         // offset

source/dnode/mnode/impl/src/mndSync.c

@@ -118,12 +118,12 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta
             transId, pTrans->createdTime, pMgmt->transId);
       mndTransExecute(pMnode, pTrans, false);
       mndReleaseTrans(pMnode, pTrans);
-      // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA);
     } else {
       mError("trans:%d, not found while execute in mnode since %s", transId, terrstr());
     }
   }
 
+  sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta);
   return 0;
 }
 
@@ -319,6 +319,7 @@ int32_t mndInitSync(SMnode *pMnode) {
     mError("failed to open sync since %s", terrstr());
     return -1;
   }
+  pMnode->pSdb->sync = pMgmt->sync;
 
   mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
   return 0;

source/dnode/mnode/impl/src/mndTrans.c

@@ -460,6 +460,8 @@ static const char *mndTransStr(ETrnStage stage) {
       return "commitAction";
     case TRN_STAGE_FINISHED:
       return "finished";
+    case TRN_STAGE_PRE_FINISH:
+      return "pre-finish";
     default:
       return "invalid";
   }
@@ -503,14 +505,15 @@ static TransCbFp mndTransGetCbFp(ETrnFunc ftype) {
 }
 
 static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
-  mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
+  mInfo("trans:%d, perform insert action, row:%p stage:%s, callfunc:1, startFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage),
+      pTrans->startFunc);
 
   if (pTrans->startFunc > 0) {
     TransCbFp fp = mndTransGetCbFp(pTrans->startFunc);
     if (fp) {
       (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
     }
-    pTrans->startFunc = 0;
+    // pTrans->startFunc = 0;
   }
 
   return 0;
@@ -546,14 +549,15 @@ static void mndTransDropData(STrans *pTrans) {
 }
 
 static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
-  mTrace("trans:%d, perform delete action, row:%p stage:%s callfunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage),
+  mInfo("trans:%d, perform delete action, row:%p stage:%s callfunc:%d, stopFunc:%d", pTrans->id, pTrans,
-         callFunc);
+        mndTransStr(pTrans->stage), callFunc, pTrans->stopFunc);
+
   if (pTrans->stopFunc > 0 && callFunc) {
     TransCbFp fp = mndTransGetCbFp(pTrans->stopFunc);
     if (fp) {
       (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
     }
-    pTrans->stopFunc = 0;
+    // pTrans->stopFunc = 0;
   }
 
   mndTransDropData(pTrans);
@@ -572,7 +576,7 @@ static void mndTransUpdateActions(SArray *pOldArray, SArray *pNewArray) {
 }
 
 static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
-  mTrace("trans:%d, perform update action, old row:%p stage:%s create:%" PRId64 ", new row:%p stage:%s create:%" PRId64,
+  mInfo("trans:%d, perform update action, old row:%p stage:%s create:%" PRId64 ", new row:%p stage:%s create:%" PRId64,
          pOld->id, pOld, mndTransStr(pOld->stage), pOld->createdTime, pNew, mndTransStr(pNew->stage),
          pNew->createdTime);
 
@@ -598,10 +602,15 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
   }
 
   if (pOld->stage == TRN_STAGE_ROLLBACK) {
-    pOld->stage = TRN_STAGE_REDO_ACTION;
+    pOld->stage = TRN_STAGE_UNDO_ACTION;
     mTrace("trans:%d, stage from rollback to undoAction since perform update action", pNew->id);
   }
 
+  if (pOld->stage == TRN_STAGE_PRE_FINISH) {
+    pOld->stage = TRN_STAGE_FINISHED;
+    mTrace("trans:%d, stage from pre-finish to finished since perform update action", pNew->id);
+  }
+
   return 0;
 }
 
@@ -866,6 +875,7 @@ int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) {
     }
   }
 
+
   if (mndCheckTransConflict(pMnode, pTrans)) {
     terrno = TSDB_CODE_MND_TRANS_CONFLICT;
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -931,6 +941,16 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
   return 0;
 }
 
+static int32_t mndTransPreFinish(SMnode *pMnode, STrans *pTrans) {
+  mInfo("trans:%d, pre-finish transaction", pTrans->id);
+  if (mndTransSync(pMnode, pTrans) != 0) {
+    mError("trans:%d, failed to pre-finish since %s", pTrans->id, terrstr());
+    return -1;
+  }
+  mInfo("trans:%d, pre-finish finished", pTrans->id);
+  return 0;
+}
+
 static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
   bool    sendRsp = false;
   int32_t code = pTrans->code;
@@ -1437,7 +1457,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) {
 
   if (code == 0) {
     pTrans->code = 0;
-    pTrans->stage = TRN_STAGE_FINISHED;
+    pTrans->stage = TRN_STAGE_FINISHED; // TRN_STAGE_PRE_FINISH is not necessary
     mInfo("trans:%d, stage from commitAction to finished", pTrans->id);
     continueExec = true;
   } else {
@@ -1455,8 +1475,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) {
   int32_t code = mndTransExecuteUndoActions(pMnode, pTrans);
 
   if (code == 0) {
-    pTrans->stage = TRN_STAGE_FINISHED;
+    pTrans->stage = TRN_STAGE_PRE_FINISH;
-    mInfo("trans:%d, stage from undoAction to finished", pTrans->id);
+    mInfo("trans:%d, stage from undoAction to pre-finish", pTrans->id);
     continueExec = true;
   } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
     mInfo("trans:%d, stage keep on undoAction since %s", pTrans->id, tstrerror(code));
@@ -1489,6 +1509,25 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
   return continueExec;
 }
 
+static bool mndTransPerfromPreFinishedStage(SMnode *pMnode, STrans *pTrans) {
+  if (mndCannotExecuteTransAction(pMnode)) return false;
+
+  bool    continueExec = true;
+  int32_t code = mndTransPreFinish(pMnode, pTrans);
+
+  if (code == 0) {
+    pTrans->stage = TRN_STAGE_FINISHED;
+    mInfo("trans:%d, stage from pre-finish to finish", pTrans->id);
+    continueExec = true;
+  } else {
+    pTrans->failedTimes++;
+    mError("trans:%d, stage keep on pre-finish since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes);
+    continueExec = false;
+  }
+
+  return continueExec;
+}
+
 static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
   bool continueExec = false;
 
@@ -1545,6 +1584,14 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) {
       case TRN_STAGE_UNDO_ACTION:
         continueExec = mndTransPerformUndoActionStage(pMnode, pTrans);
         break;
+      case TRN_STAGE_PRE_FINISH:
+        if (isLeader) {
+          continueExec = mndTransPerfromPreFinishedStage(pMnode, pTrans);
+        } else {
+          mInfo("trans:%d, can not pre-finish since not leader", pTrans->id);
+          continueExec = false;
+        }
+        break;
       case TRN_STAGE_FINISHED:
         continueExec = mndTransPerfromFinishedStage(pMnode, pTrans);
         break;
@@ -1645,8 +1692,6 @@ void mndTransPullup(SMnode *pMnode) {
     }
     mndReleaseTrans(pMnode, pTrans);
   }
-
-  sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA);
   taosArrayDestroy(pArray);
 }
 

source/dnode/mnode/impl/src/mndVgroup.c

@@ -2209,11 +2209,12 @@ _OVER:
 
 bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }
 
-static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen,
+static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, int64_t compactTs,
-                                     int64_t compactTs) {
+                                     STimeWindow tw) {
   SCompactVnodeReq compactReq = {0};
   compactReq.dbUid = pDb->uid;
   compactReq.compactStartTime = compactTs;
+  compactReq.tw = tw;
   tstrncpy(compactReq.db, pDb->name, TSDB_DB_FNAME_LEN);
 
   mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
@@ -2239,13 +2240,13 @@ static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgrou
   return pReq;
 }
 
-static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
+static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs,
-                                        int64_t compactTs) {
+                                        STimeWindow tw) {
   STransAction action = {0};
   action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
   int32_t contLen = 0;
-  void   *pReq = mndBuildCompactVnodeReq(pMnode, pDb, pVgroup, &contLen, compactTs);
+  void   *pReq = mndBuildCompactVnodeReq(pMnode, pDb, pVgroup, &contLen, compactTs, tw);
   if (pReq == NULL) return -1;
 
   action.pCont = pReq;
@@ -2260,7 +2261,8 @@ static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *
   return 0;
 }
 
-int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs) {
+int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs,
-  if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) return -1;
+                                    STimeWindow tw) {
+  if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs, tw) != 0) return -1;
   return 0;
 }

source/dnode/mnode/impl/test/profile/profile.cpp

@@ -32,13 +32,14 @@ TEST_F(MndTestProfile, 01_ConnectMsg) {
   connectReq.pid = 1234;
 
   char passwd[] = "taosdata";
-  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
+  char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
   taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
 
   strcpy(connectReq.app, "mnode_test_profile");
   strcpy(connectReq.db, "");
   strcpy(connectReq.user, "root");
   strcpy(connectReq.passwd, secretEncrypt);
+  strcpy(connectReq.sVer, version);
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = rpcMallocCont(contLen);
@@ -66,7 +67,7 @@ TEST_F(MndTestProfile, 01_ConnectMsg) {
 
 TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) {
   char passwd[] = "taosdata";
-  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
+  char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
   taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
 
   SConnectReq connectReq = {0};
@@ -75,6 +76,7 @@ TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) {
   strcpy(connectReq.db, "invalid_db");
   strcpy(connectReq.user, "root");
   strcpy(connectReq.passwd, secretEncrypt);
+  strcpy(connectReq.sVer, version);
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = rpcMallocCont(contLen);

source/dnode/mnode/impl/test/show/show.cpp

@@ -55,7 +55,7 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
 
 TEST_F(MndTestShow, 03_ShowMsg_Conn) {
   char passwd[] = "taosdata";
-  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
+  char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
   taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
 
   SConnectReq connectReq = {0};
@@ -64,6 +64,7 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) {
   strcpy(connectReq.db, "");
   strcpy(connectReq.user, "root");
   strcpy(connectReq.passwd, secretEncrypt);
+  strcpy(connectReq.sVer, version);
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = rpcMallocCont(contLen);

source/dnode/mnode/sdb/inc/sdb.h

@@ -37,8 +37,6 @@ extern "C" {
 #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
 // clang-format on
 
-#define SDB_WRITE_DELTA 20
-
 #define SDB_GET_VAL(pData, dataPos, val, pos, func, type) \
   {                                                       \
     if (func(pRaw, dataPos, val) != 0) {                  \

source/dnode/mnode/sdb/src/sdb.c

@@ -53,7 +53,6 @@ SSdb *sdbInit(SSdbOpt *pOption) {
   }
 
   pSdb->pWal = pOption->pWal;
-  pSdb->sync = pOption->sync;
   pSdb->applyIndex = -1;
   pSdb->applyTerm = -1;
   pSdb->applyConfig = -1;

source/dnode/mnode/sdb/src/sdbFile.c

@@ -472,10 +472,7 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) {
 
   taosThreadMutexLock(&pSdb->filelock);
   if (pSdb->pWal != NULL) {
-    // code = walBeginSnapshot(pSdb->pWal, pSdb->applyIndex, 0);
+    if (pSdb->sync > 0) {
-    if (pSdb->sync == 0) {
-      code = 0;
-    } else {
       code = syncBeginSnapshot(pSdb->sync, pSdb->applyIndex);
     }
   }
@@ -484,11 +481,7 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) {
   }
   if (code == 0) {
     if (pSdb->pWal != NULL) {
-      // code = walEndSnapshot(pSdb->pWal);
+      if (pSdb->sync > 0) {
-
-      if (pSdb->sync == 0) {
-        code = 0;
-      } else {
         code = syncEndSnapshot(pSdb->sync);
       }
     }

source/dnode/mnode/sdb/src/sdbHash.c

@@ -160,6 +160,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   if (insertFp != NULL) {
     code = (*insertFp)(pSdb, pRow->pObj);
     if (code != 0) {
+      if (terrno == 0) terrno = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
       code = terrno;
       taosHashRemove(hash, pRow->pObj, keySize);
       sdbFreeRow(pSdb, pRow, false);

source/dnode/vnode/CMakeLists.txt

@@ -14,7 +14,6 @@ target_sources(
     "src/vnd/vnodeSvr.c"
     "src/vnd/vnodeSync.c"
     "src/vnd/vnodeSnapshot.c"
-    "src/vnd/vnodeCompact.c"
     "src/vnd/vnodeRetention.c"
 
     # meta
@@ -53,7 +52,6 @@ target_sources(
     "src/tsdb/tsdbCacheRead.c"
     "src/tsdb/tsdbRetention.c"
     "src/tsdb/tsdbDiskData.c"
-    "src/tsdb/tsdbCompact.c"
     "src/tsdb/tsdbMergeTree.c"
     "src/tsdb/tsdbDataIter.c"
 
@@ -69,10 +67,20 @@ target_sources(
     "src/tq/tqSnapshot.c"
     "src/tq/tqOffsetSnapshot.c"
 )
+
+IF (TD_VNODE_PLUGINS)
+  target_sources(
+    vnode
+    PRIVATE
+    ${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/tsdbCompact.c
+    ${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/vnodeCompact.c
+  )
+ENDIF ()
+
 target_include_directories(
     vnode
     PUBLIC "inc"
-    PRIVATE "src/inc"
+    PUBLIC "src/inc"
     PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
 )
 target_link_libraries(

source/dnode/vnode/inc/vnode.h

@@ -264,7 +264,7 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
 int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
 int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
 
-int32_t tqSeekVer(STqReader *pReader, int64_t ver);
+int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char* id);
 int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret);
 
 int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);

source/dnode/vnode/src/inc/tq.h

@@ -67,7 +67,7 @@ typedef struct {
 // tqExec
 
 typedef struct {
-  char* qmsg;
+  char* qmsg;   // SubPlanToString
 } STqExecCol;
 
 typedef struct {

source/dnode/vnode/src/inc/tsdb.h

@@ -766,6 +766,7 @@ typedef struct SCacheRowsReader {
   TdThreadMutex      readerMutex;
   SVnode            *pVnode;
   STSchema          *pSchema;
+  STSchema          *pCurrSchema;
   uint64_t           uid;
   uint64_t           suid;
   char             **transferBuf;  // todo remove it soon

source/dnode/vnode/src/inc/vnd.h

@@ -106,10 +106,6 @@ int32_t vnodeSyncCommit(SVnode* pVnode);
 int32_t vnodeAsyncCommit(SVnode* pVnode);
 bool    vnodeShouldRollback(SVnode* pVnode);
 
-// vnodeCompact.c
-int32_t vnodeAsyncCompact(SVnode* pVnode);
-int32_t vnodeSyncCompact(SVnode* pVnode);
-
 // vnodeSync.c
 int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
 int32_t vnodeSyncStart(SVnode* pVnode);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -457,9 +457,10 @@ struct SCommitInfo {
 };
 
 struct SCompactInfo {
-  SVnode* pVnode;
+  SVnode*     pVnode;
-  int32_t flag;
+  int32_t     flag;
-  int64_t commitID;
+  int64_t     commitID;
+  STimeWindow tw;
 };
 
 #ifdef __cplusplus

source/dnode/vnode/src/tq/tq.c

@@ -488,7 +488,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
 
   // 2. check rebalance
   if (pHandle->consumerId != consumerId) {
-    tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
+    tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
             consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId);
     terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
     return -1;
@@ -575,7 +575,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       return -1;
     }
 
-#if 1
     // till now, all data has been rsp to consumer, new data needs to push client once arrived.
     if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
         dataRsp.reqOffset.version == dataRsp.rspOffset.version) {
@@ -597,7 +596,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
         return 0;
       }
     }
-#endif
     taosWUnLockLatch(&pTq->pushLock);
 
     if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) {
@@ -613,10 +611,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   // for taosx
-  /*A(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN);*/
-
   SMqMetaRsp metaRsp = {0};
-
   STaosxRsp taosxRsp = {0};
   tqInitTaosxRsp(&taosxRsp, &req);
 
@@ -827,6 +822,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
     pHandle = &tqHandle;
     /*taosInitRWLatch(&pExec->lock);*/
 
+    uint64_t oldConsumerId = pHandle->consumerId;
     memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
     pHandle->consumerId = req.newConsumerId;
     pHandle->epoch = -1;
@@ -889,13 +885,16 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
                        (SSnapContext**)(&handle.sContext));
       pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
     }
+
     taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
-    tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, pHandle->consumerId);
+    tqDebug("try to persist handle %s consumer:0x%" PRIx64" , old consumer:0x%"PRIx64, req.subKey, pHandle->consumerId,
+            oldConsumerId);
     if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
       return -1;
     }
   } else {
     // TODO handle qmsg and exec modification
+    tqInfo("update the consumer info, old consumer id:0x%"PRIx64", new Id:0x%"PRIx64, pHandle->consumerId, req.newConsumerId);
     atomic_store_32(&pHandle->epoch, -1);
     atomic_store_64(&pHandle->consumerId, req.newConsumerId);
     atomic_add_fetch_32(&pHandle->epoch, 1);
@@ -903,6 +902,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
     if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
       qStreamCloseTsdbReader(pHandle->execHandle.task);
     }
+
     if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
       return -1;
     }

source/dnode/vnode/src/tq/tqExec.c

@@ -147,7 +147,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
     }
     tqDebug("tmqsnap task execute end, get %p", pDataBlock);
 
-    if (pDataBlock != NULL) {
+    if (pDataBlock != NULL && pDataBlock->info.rows > 0) {
       if (pRsp->withTbName) {
         if (pOffset->type == TMQ_OFFSET__LOG) {
           int64_t uid = pExec->pExecReader->lastBlkUid;

source/dnode/vnode/src/tq/tqRead.c

@@ -290,14 +290,14 @@ void tqCloseReader(STqReader* pReader) {
   taosMemoryFree(pReader);
 }
 
-int32_t tqSeekVer(STqReader* pReader, int64_t ver) {
+int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
   // todo set the correct vgId
-  tqDebug("tmq poll: vgId:%d wal seek to version:%"PRId64, 0, ver);
+  tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id);
   if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
-    tqError("tmq poll: wal reader failed to seek to ver:%"PRId64, ver);
+    tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
     return -1;
   } else {
-    tqDebug("tmq poll: wal reader seek to ver:%"PRId64, ver);
+    tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);
     return 0;
   }
 }
@@ -307,36 +307,26 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
 
   while (1) {
     if (!fromProcessedMsg) {
-      SWalReader* pWalReader = pReader->pWalReader;
+      if (walNextValidMsg(pReader->pWalReader) < 0) {
-
+//        pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped;
-      if (walNextValidMsg(pWalReader) < 0) {
+        if(pReader->pWalReader->curInvalid == 0){
-        pReader->ver = pWalReader->curVersion - (pWalReader->curInvalid | pWalReader->curStopped);
+          pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped;
+        }else{
+          pReader->ver = walGetLastVer(pReader->pWalReader->pWal);
+        }
         ret->offset.type = TMQ_OFFSET__LOG;
+
         ret->offset.version = pReader->ver;
         ret->fetchType = FETCH_TYPE__NONE;
-        tqDebug("return offset %" PRId64 ", no more valid", ret->offset.version);
+        tqDebug("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version);
         return -1;
       }
 
-      void*   body = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
+      void*   body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
-      int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
+      int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
-      int64_t ver = pWalReader->pHead->head.version;
+      int64_t ver = pReader->pWalReader->pHead->head.version;
 
-      tqDebug("tmq poll: extract submit msg from wal, version:%"PRId64" len:%d", ver, bodyLen);
-
-#if 0
-      if (pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) {
-        // TODO do filter
-        ret->fetchType = FETCH_TYPE__META;
-        ret->meta = pWalReader->pHead->head.body;
-        return 0;
-      } else {
-#endif
       tqReaderSetSubmitReq2(pReader, body, bodyLen, ver);
-      /*tqReaderSetDataMsg(pReader, body, pWalReader->pHead->head.version);*/
-#if 0
-      }
-#endif
     }
 
     while (tqNextDataBlock2(pReader)) {

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -257,9 +257,6 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TS
 
         SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
         if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
-          SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
-          taosMemoryFree(pLastCol->colVal.value.pData);
-
           lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
           if (lastCol.colVal.value.pData == NULL) {
             terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -378,9 +375,6 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb
 
         SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
         if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
-          SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
-          taosMemoryFree(pLastCol->colVal.value.pData);
-
           lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
           if (lastCol.colVal.value.pData == NULL) {
             terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -1265,17 +1259,58 @@ _err:
   return code;
 }
 
-static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppColArray, SCacheRowsReader *pr) {
+static int32_t initLastColArray(STSchema *pTSchema, SArray **ppColArray) {
-  int32_t code = 0;
+  SArray *pColArray = taosArrayInit(pTSchema->numOfCols, sizeof(SLastCol));
+  if (NULL == pColArray) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
 
+  for (int32_t i = 0; i < pTSchema->numOfCols; ++i) {
+    SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[i].colId, pTSchema->columns[i].type)};
+    taosArrayPush(pColArray, &col);
+  }
+  *ppColArray = pColArray;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) {
+  int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols;
+  *ppDst = taosMemoryMalloc(len);
+  if (NULL == *ppDst) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  memcpy(*ppDst, pSrc, len);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) {
+  if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) {
+    return cloneTSchema(pReader->pSchema, &pReader->pCurrSchema);
+  }
+
+  if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  taosMemoryFreeClear(pReader->pCurrSchema);
+  return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pCurrSchema);
+}
+
+static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppColArray, SCacheRowsReader *pr) {
   STSchema *pTSchema = pr->pSchema;  // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
-  int16_t   nCol = pTSchema->numOfCols;
+  int16_t   nLastCol = pTSchema->numOfCols;
   int16_t   iCol = 0;
   int16_t   noneCol = 0;
   bool      setNoneCol = false;
-  SArray   *pColArray = taosArrayInit(nCol, sizeof(SLastCol));
+  bool      hasRow = false;
+  SArray   *pColArray = NULL;
   SColVal  *pColVal = &(SColVal){0};
 
+  int32_t code = initLastColArray(pTSchema, &pColArray);
+  if (TSDB_CODE_SUCCESS != code) {
+    return code;
+  }
+
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
@@ -1290,6 +1325,15 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
       break;
     }
 
+    hasRow = true;
+
+    code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid);
+    if (TSDB_CODE_SUCCESS != code) {
+      goto _err;
+    }
+    pTSchema = pr->pCurrSchema;
+    int16_t nCol = pTSchema->numOfCols;
+
     TSKEY rowTs = TSDBROW_TS(pRow);
 
     if (lastRowTs == TSKEY_MAX) {
@@ -1297,29 +1341,27 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
       STColumn *pTColumn = &pTSchema->columns[0];
 
       *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
-      if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
+      taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal});
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _err;
-      }
 
       for (iCol = 1; iCol < nCol; ++iCol) {
+        if (iCol >= nLastCol) {
+          break;
+        }
+        SLastCol *pCol = taosArrayGet(pColArray, iCol);
+        if (pCol->colVal.cid != pTSchema->columns[iCol].colId) {
+          continue;
+        }
         tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
 
-        SLastCol lastCol = {.ts = lastRowTs, .colVal = *pColVal};
+        *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal};
         if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) {
-          lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData);
+          pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
-          if (lastCol.colVal.value.pData == NULL) {
+          if (pCol->colVal.value.pData == NULL) {
             terrno = TSDB_CODE_OUT_OF_MEMORY;
             code = TSDB_CODE_OUT_OF_MEMORY;
             goto _err;
           }
-          memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
-        }
-
-        if (taosArrayPush(pColArray, &lastCol) == NULL) {
-          code = TSDB_CODE_OUT_OF_MEMORY;
-          goto _err;
         }
 
         if (COL_VAL_IS_NONE(pColVal) && !setNoneCol) {
@@ -1376,6 +1418,9 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
   //*ppColArray = NULL;
   // taosArrayDestroy(pColArray);
   //} else {
+  if (!hasRow) {
+    taosArrayClear(pColArray);
+  }
   *ppColArray = pColArray;
   //}
 
@@ -1391,16 +1436,19 @@ _err:
 }
 
 static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr) {
-  int32_t code = 0;
-
   STSchema *pTSchema = pr->pSchema;  // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
-  int16_t   nCol = pTSchema->numOfCols;
+  int16_t   nLastCol = pTSchema->numOfCols;
-  int16_t   iCol = 0;
   int16_t   noneCol = 0;
   bool      setNoneCol = false;
-  SArray   *pColArray = taosArrayInit(nCol, sizeof(SLastCol));
+  bool      hasRow = false;
+  SArray   *pColArray = NULL;
   SColVal  *pColVal = &(SColVal){0};
 
+  int32_t code = initLastColArray(pTSchema, &pColArray);
+  if (TSDB_CODE_SUCCESS != code) {
+    return code;
+  }
+
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
@@ -1415,6 +1463,15 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
       break;
     }
 
+    hasRow = true;
+
+    code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid);
+    if (TSDB_CODE_SUCCESS != code) {
+      goto _err;
+    }
+    pTSchema = pr->pCurrSchema;
+    int16_t nCol = pTSchema->numOfCols;
+
     TSKEY rowTs = TSDBROW_TS(pRow);
 
     if (lastRowTs == TSKEY_MAX) {
@@ -1422,28 +1479,27 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
       STColumn *pTColumn = &pTSchema->columns[0];
 
       *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
-      if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
+      taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal});
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _err;
-      }
 
-      for (iCol = 1; iCol < nCol; ++iCol) {
+      for (int16_t iCol = 1; iCol < nCol; ++iCol) {
+        if (iCol >= nLastCol) {
+          break;
+        }
+        SLastCol *pCol = taosArrayGet(pColArray, iCol);
+        if (pCol->colVal.cid != pTSchema->columns[iCol].colId) {
+          continue;
+        }
         tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
 
-        SLastCol lastCol = {.ts = lastRowTs, .colVal = *pColVal};
+        *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal};
         if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) {
-          lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData);
+          pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
-          if (lastCol.colVal.value.pData == NULL) {
+          if (pCol->colVal.value.pData == NULL) {
             terrno = TSDB_CODE_OUT_OF_MEMORY;
             code = TSDB_CODE_OUT_OF_MEMORY;
             goto _err;
           }
-          memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
-        }
-
-        if (taosArrayPush(pColArray, &lastCol) == NULL) {
-          code = TSDB_CODE_OUT_OF_MEMORY;
-          goto _err;
         }
 
         if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
@@ -1461,10 +1517,16 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
 
     // merge into pColArray
     setNoneCol = false;
-    for (iCol = noneCol; iCol < nCol; ++iCol) {
+    for (int16_t iCol = noneCol; iCol < nCol; ++iCol) {
+      if (iCol >= nLastCol) {
+        break;
+      }
       // high version's column value
       SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol);
-      SColVal  *tColVal = &lastColVal->colVal;
+      if (lastColVal->colVal.cid != pTSchema->columns[iCol].colId) {
+        continue;
+      }
+      SColVal *tColVal = &lastColVal->colVal;
 
       tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
       if (!COL_VAL_IS_VALUE(tColVal) && COL_VAL_IS_VALUE(pColVal)) {
@@ -1494,6 +1556,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
   //*ppLastArray = NULL;
   // taosArrayDestroy(pColArray);
   //} else {
+  if (!hasRow) {
+    taosArrayClear(pColArray);
+  }
   *ppLastArray = pColArray;
   //}
 

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -38,16 +38,17 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
         *(int64_t*)p->buf = pColVal->ts;
         allNullRow = false;
       } else {
-        int32_t   slotId = slotIds[i];
+        int32_t slotId = slotIds[i];
-        SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, slotId);
-
         // add check for null value, caused by the modification of table schema (new column added).
-        if (pColVal == NULL) {
+        if (slotId >= taosArrayGetSize(pRow)) {
           p->ts = 0;
           p->isNull = true;
+          colDataSetNULL(pColInfoData, numOfRows);
           continue;
         }
 
+        SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, slotId);
+
         p->ts = pColVal->ts;
         p->isNull = !COL_VAL_IS_VALUE(&pColVal->colVal);
         allNullRow = p->isNull & allNullRow;
@@ -79,7 +80,12 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
         SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0);
         colDataSetVal(pColInfoData, numOfRows, (const char*)&pColVal->ts, false);
       } else {
-        int32_t   slotId = slotIds[i];
+        int32_t slotId = slotIds[i];
+        // add check for null value, caused by the modification of table schema (new column added).
+        if (slotId >= taosArrayGetSize(pRow)) {
+          colDataSetNULL(pColInfoData, numOfRows);
+          continue;
+        }
         SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, slotId);
         SColVal*  pVal = &pColVal->colVal;
 
@@ -209,6 +215,8 @@ void* tsdbCacherowsReaderClose(void* pReader) {
     taosMemoryFree(p->pSchema);
   }
 
+  taosMemoryFree(p->pCurrSchema);
+
   destroyLastBlockLoadInfo(p->pLoadInfo);
 
   taosMemoryFree((void*)p->idstr);
@@ -290,7 +298,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
   }
 
   for (int32_t j = 0; j < pr->numOfCols; ++j) {
-    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE);
+    pRes[j] = taosMemoryCalloc(
+        1, sizeof(SFirstLastRes) + pr->pSchema->columns[-1 == slotIds[j] ? 0 : slotIds[j]].bytes + VARSTR_HEADER_SIZE);
     SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);
     p->ts = INT64_MIN;
   }
@@ -303,7 +312,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
   for (int32_t i = 0; i < pr->pSchema->numOfCols; ++i) {
     struct STColumn* pCol = &pr->pSchema->columns[i];
-    SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type};
+    SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
 
     if (IS_VAR_DATA_TYPE(pCol->type)) {
       p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));

source/dnode/vnode/src/tsdb/tsdbCompact.c

@@ -1,664 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "tsdb.h"
-
-extern int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo);
-extern int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg);
-extern int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg);
-
-typedef struct {
-  STsdb  *pTsdb;
-  int64_t commitID;
-  int8_t  cmprAlg;
-  int32_t maxRows;
-  int32_t minRows;
-
-  STsdbFS fs;
-
-  int32_t  fid;
-  TABLEID  tbid;
-  SSkmInfo tbSkm;
-
-  // Tombstone
-  SDelFReader *pDelFReader;
-  SArray      *aDelIdx;   // SArray<SDelIdx>
-  SArray      *aDelData;  // SArray<SDelData>
-  SArray      *aSkyLine;  // SArray<TSDBKEY>
-  int32_t      iDelIdx;
-  int32_t      iSkyLine;
-  TSDBKEY     *pDKey;
-  TSDBKEY      dKey;
-
-  // Reader
-  SDataFReader   *pReader;
-  STsdbDataIter2 *iterList;  // list of iterators
-  STsdbDataIter2 *pIter;
-  SRBTree         rbt;
-
-  // Writer
-  SDataFWriter *pWriter;
-  SArray       *aBlockIdx;  // SArray<SBlockIdx>
-  SMapData      mDataBlk;   // SMapData<SDataBlk>
-  SArray       *aSttBlk;    // SArray<SSttBlk>
-  SBlockData    bData;
-  SBlockData    sData;
-} STsdbCompactor;
-
-static int32_t tsdbAbortCompact(STsdbCompactor *pCompactor) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  STsdb *pTsdb = pCompactor->pTsdb;
-  code = tsdbFSRollback(pTsdb);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
-  } else {
-    tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__);
-  }
-  return code;
-}
-
-static int32_t tsdbCompactWriteTableDataStart(STsdbCompactor *pCompactor, TABLEID *pId) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  pCompactor->tbid = *pId;
-
-  // tombstone
-  for (;;) {
-    if (pCompactor->iDelIdx >= taosArrayGetSize(pCompactor->aDelIdx)) {
-      pCompactor->pDKey = NULL;
-      break;
-    }
-
-    SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(pCompactor->aDelIdx, pCompactor->iDelIdx);
-    int32_t  c = tTABLEIDCmprFn(pDelIdx, &pCompactor->tbid);
-    if (c < 0) {
-      pCompactor->iDelIdx++;
-    } else if (c == 0) {
-      pCompactor->iDelIdx++;
-
-      code = tsdbReadDelData(pCompactor->pDelFReader, pDelIdx, pCompactor->aDelData);
-      TSDB_CHECK_CODE(code, lino, _exit);
-
-      code = tsdbBuildDeleteSkyline(pCompactor->aDelData, 0, taosArrayGetSize(pCompactor->aDelData) - 1,
-                                    pCompactor->aSkyLine);
-      TSDB_CHECK_CODE(code, lino, _exit);
-
-      pCompactor->iSkyLine = 0;
-      if (pCompactor->iSkyLine < taosArrayGetSize(pCompactor->aSkyLine)) {
-        TSDBKEY *pKey = (TSDBKEY *)taosArrayGet(pCompactor->aSkyLine, pCompactor->iSkyLine);
-
-        pCompactor->dKey.version = 0;
-        pCompactor->dKey.ts = pKey->ts;
-        pCompactor->pDKey = &pCompactor->dKey;
-      } else {
-        pCompactor->pDKey = NULL;
-      }
-      break;
-    } else {
-      pCompactor->pDKey = NULL;
-      break;
-    }
-  }
-
-  // writer
-  code = tsdbUpdateTableSchema(pCompactor->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pCompactor->tbSkm);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  tMapDataReset(&pCompactor->mDataBlk);
-
-  code = tBlockDataInit(&pCompactor->bData, pId, pCompactor->tbSkm.pTSchema, NULL, 0);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  if (!TABLE_SAME_SCHEMA(pCompactor->sData.suid, pCompactor->sData.uid, pId->suid, pId->uid)) {
-    if (pCompactor->sData.nRow > 0) {
-      code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg);
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    TABLEID tbid = {.suid = pId->suid, .uid = pId->suid ? 0 : pId->uid};
-    code = tBlockDataInit(&pCompactor->sData, &tbid, pCompactor->tbSkm.pTSchema, NULL, 0);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
-              tstrerror(code));
-  } else {
-    tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pId->suid,
-              pId->uid);
-  }
-  return code;
-}
-
-static int32_t tsdbCompactWriteTableDataEnd(STsdbCompactor *pCompactor) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  if (pCompactor->bData.nRow > 0) {
-    if (pCompactor->bData.nRow < pCompactor->minRows) {
-      for (int32_t iRow = 0; iRow < pCompactor->bData.nRow; iRow++) {
-        code = tBlockDataAppendRow(&pCompactor->sData, &tsdbRowFromBlockData(&pCompactor->bData, iRow), NULL,
-                                   pCompactor->tbid.uid);
-        TSDB_CHECK_CODE(code, lino, _exit);
-
-        if (pCompactor->sData.nRow >= pCompactor->maxRows) {
-          code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg);
-          TSDB_CHECK_CODE(code, lino, _exit);
-        }
-      }
-      tBlockDataClear(&pCompactor->bData);
-    } else {
-      code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg);
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-  }
-
-  if (pCompactor->mDataBlk.nItem > 0) {
-    SBlockIdx *pBlockIdx = (SBlockIdx *)taosArrayReserve(pCompactor->aBlockIdx, 1);
-    if (pBlockIdx == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    pBlockIdx->suid = pCompactor->tbid.suid;
-    pBlockIdx->uid = pCompactor->tbid.uid;
-
-    code = tsdbWriteDataBlk(pCompactor->pWriter, &pCompactor->mDataBlk, pBlockIdx);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
-              tstrerror(code));
-  } else {
-    tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__,
-              pCompactor->tbid.suid, pCompactor->tbid.uid);
-  }
-  return code;
-}
-
-static bool tsdbCompactRowIsDeleted(STsdbCompactor *pCompactor, TSDBROW *pRow) {
-  TSDBKEY  tKey = TSDBROW_KEY(pRow);
-  TSDBKEY *aKey = (TSDBKEY *)TARRAY_DATA(pCompactor->aSkyLine);
-  int32_t  nKey = TARRAY_SIZE(pCompactor->aSkyLine);
-
-  if (tKey.ts > pCompactor->pDKey->ts) {
-    do {
-      pCompactor->pDKey->version = aKey[pCompactor->iSkyLine].version;
-      pCompactor->iSkyLine++;
-      if (pCompactor->iSkyLine < nKey) {
-        pCompactor->dKey.ts = aKey[pCompactor->iSkyLine].ts;
-      } else {
-        if (pCompactor->pDKey->version == 0) {
-          pCompactor->pDKey = NULL;
-          return false;
-        } else {
-          pCompactor->pDKey->ts = INT64_MAX;
-        }
-      }
-    } while (tKey.ts > pCompactor->pDKey->ts);
-  }
-
-  if (tKey.ts < pCompactor->pDKey->ts) {
-    if (tKey.version > pCompactor->pDKey->version) {
-      return false;
-    } else {
-      return true;
-    }
-  } else if (tKey.ts == pCompactor->pDKey->ts) {
-    ASSERT(pCompactor->iSkyLine < nKey);
-    if (tKey.version > TMAX(pCompactor->pDKey->version, aKey[pCompactor->iSkyLine].version)) {
-      return false;
-    } else {
-      return true;
-    }
-  }
-
-  return false;
-}
-
-static int32_t tsdbCompactWriteTableData(STsdbCompactor *pCompactor, SRowInfo *pRowInfo) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  // start a new table data write if need
-  if (pRowInfo == NULL || pRowInfo->uid != pCompactor->tbid.uid) {
-    if (pCompactor->tbid.uid) {
-      code = tsdbCompactWriteTableDataEnd(pCompactor);
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    if (pRowInfo == NULL) {
-      if (pCompactor->sData.nRow > 0) {
-        code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg);
-        TSDB_CHECK_CODE(code, lino, _exit);
-      }
-      return code;
-    }
-
-    code = tsdbCompactWriteTableDataStart(pCompactor, (TABLEID *)pRowInfo);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  // check if row is deleted
-  if (pCompactor->pDKey && tsdbCompactRowIsDeleted(pCompactor, &pRowInfo->row)) goto _exit;
-
-  if (tBlockDataTryUpsertRow(&pCompactor->bData, &pRowInfo->row, pRowInfo->uid) > pCompactor->maxRows) {
-    code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  code = tBlockDataUpsertRow(&pCompactor->bData, &pRowInfo->row, NULL, pRowInfo->uid);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
-              tstrerror(code));
-  } else if (pRowInfo) {
-    tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " ts:%" PRId64 " version:%" PRId64,
-              TD_VID(pCompactor->pTsdb->pVnode), __func__, pRowInfo->suid, pRowInfo->uid, TSDBROW_TS(&pRowInfo->row),
-              TSDBROW_VERSION(&pRowInfo->row));
-  }
-  return code;
-}
-
-static bool tsdbCompactTableIsDropped(STsdbCompactor *pCompactor) {
-  SMetaInfo info;
-
-  if (pCompactor->pIter->rowInfo.uid == pCompactor->tbid.uid) return false;
-  if (metaGetInfo(pCompactor->pTsdb->pVnode->pMeta, pCompactor->pIter->rowInfo.uid, &info, NULL)) {
-    return true;
-  }
-  return false;
-}
-static int32_t tsdbCompactNextRow(STsdbCompactor *pCompactor, SRowInfo **ppRowInfo) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  for (;;) {
-    if (pCompactor->pIter) {
-      code = tsdbDataIterNext2(pCompactor->pIter, NULL);
-      TSDB_CHECK_CODE(code, lino, _exit);
-
-      if (pCompactor->pIter->rowInfo.suid == 0 && pCompactor->pIter->rowInfo.uid == 0) {
-        pCompactor->pIter = NULL;
-      } else {
-        SRBTreeNode *pNode = tRBTreeMin(&pCompactor->rbt);
-        if (pNode) {
-          int32_t c = tsdbDataIterCmprFn(&pCompactor->pIter->rbtn, pNode);
-          if (c > 0) {
-            tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
-            pCompactor->pIter = NULL;
-          } else if (c == 0) {
-            ASSERT(0);
-          }
-        }
-      }
-    }
-
-    if (pCompactor->pIter == NULL) {
-      SRBTreeNode *pNode = tRBTreeDropMin(&pCompactor->rbt);
-      if (pNode) {
-        pCompactor->pIter = TSDB_RBTN_TO_DATA_ITER(pNode);
-      }
-    }
-
-    if (pCompactor->pIter) {
-      if (tsdbCompactTableIsDropped(pCompactor)) {
-        TABLEID tbid = {.suid = pCompactor->pIter->rowInfo.suid, .uid = pCompactor->pIter->rowInfo.uid};
-        tRBTreeClear(&pCompactor->rbt);
-        for (pCompactor->pIter = pCompactor->iterList; pCompactor->pIter; pCompactor->pIter = pCompactor->pIter->next) {
-          code = tsdbDataIterNext2(pCompactor->pIter,
-                                   &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_TABLEID, .tbid = tbid});
-          TSDB_CHECK_CODE(code, lino, _exit);
-
-          if (pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid) {
-            tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
-          }
-        }
-      } else {
-        *ppRowInfo = &pCompactor->pIter->rowInfo;
-        break;
-      }
-    } else {
-      *ppRowInfo = NULL;
-      break;
-    }
-  }
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
-              tstrerror(code));
-  }
-  return code;
-}
-
-static int32_t tsdbCompactFileSetStart(STsdbCompactor *pCompactor, SDFileSet *pSet) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  pCompactor->fid = pSet->fid;
-  pCompactor->tbid = (TABLEID){0};
-
-  /* tombstone */
-  pCompactor->iDelIdx = 0;
-
-  /* reader */
-  code = tsdbDataFReaderOpen(&pCompactor->pReader, pCompactor->pTsdb, pSet);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  code = tsdbOpenDataFileDataIter(pCompactor->pReader, &pCompactor->pIter);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  tRBTreeCreate(&pCompactor->rbt, tsdbDataIterCmprFn);
-  if (pCompactor->pIter) {
-    pCompactor->pIter->next = pCompactor->iterList;
-    pCompactor->iterList = pCompactor->pIter;
-
-    code = tsdbDataIterNext2(pCompactor->pIter, NULL);
-    TSDB_CHECK_CODE(code, lino, _exit);
-
-    ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid);
-    tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
-  }
-
-  for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
-    code = tsdbOpenSttFileDataIter(pCompactor->pReader, iStt, &pCompactor->pIter);
-    TSDB_CHECK_CODE(code, lino, _exit);
-
-    if (pCompactor->pIter) {
-      pCompactor->pIter->next = pCompactor->iterList;
-      pCompactor->iterList = pCompactor->pIter;
-
-      code = tsdbDataIterNext2(pCompactor->pIter, NULL);
-      TSDB_CHECK_CODE(code, lino, _exit);
-
-      ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid);
-      tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn);
-    }
-  }
-  pCompactor->pIter = NULL;
-
-  /* writer */
-  code = tsdbDataFWriterOpen(&pCompactor->pWriter, pCompactor->pTsdb,
-                             &(SDFileSet){.fid = pCompactor->fid,
-                                          .diskId = pSet->diskId,
-                                          .pHeadF = &(SHeadFile){.commitID = pCompactor->commitID},
-                                          .pDataF = &(SDataFile){.commitID = pCompactor->commitID},
-                                          .pSmaF = &(SSmaFile){.commitID = pCompactor->commitID},
-                                          .nSttF = 1,
-                                          .aSttF = {&(SSttFile){.commitID = pCompactor->commitID}}});
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  if (pCompactor->aBlockIdx) {
-    taosArrayClear(pCompactor->aBlockIdx);
-  } else if ((pCompactor->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  tMapDataReset(&pCompactor->mDataBlk);
-
-  if (pCompactor->aSttBlk) {
-    taosArrayClear(pCompactor->aSttBlk);
-  } else if ((pCompactor->aSttBlk = taosArrayInit(0, sizeof(SSttBlk))) == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  tBlockDataReset(&pCompactor->bData);
-  tBlockDataReset(&pCompactor->sData);
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
-              tstrerror(code), pCompactor->fid);
-  } else {
-    tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid);
-  }
-  return code;
-}
-
-static int32_t tsdbCompactFileSetEnd(STsdbCompactor *pCompactor) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  ASSERT(pCompactor->bData.nRow == 0);
-  ASSERT(pCompactor->sData.nRow == 0);
-
-  /* update files */
-  code = tsdbWriteSttBlk(pCompactor->pWriter, pCompactor->aSttBlk);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  code = tsdbWriteBlockIdx(pCompactor->pWriter, pCompactor->aBlockIdx);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  code = tsdbUpdateDFileSetHeader(pCompactor->pWriter);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  code = tsdbFSUpsertFSet(&pCompactor->fs, &pCompactor->pWriter->wSet);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  code = tsdbDataFWriterClose(&pCompactor->pWriter, 1);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  code = tsdbDataFReaderClose(&pCompactor->pReader);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  /* do clear */
-  while ((pCompactor->pIter = pCompactor->iterList) != NULL) {
-    pCompactor->iterList = pCompactor->pIter->next;
-    tsdbCloseDataIter2(pCompactor->pIter);
-  }
-
-  tBlockDataReset(&pCompactor->bData);
-  tBlockDataReset(&pCompactor->sData);
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
-              tstrerror(code), pCompactor->fid);
-  } else {
-    tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid);
-  }
-  return code;
-}
-
-static int32_t tsdbCompactFileSet(STsdbCompactor *pCompactor, SDFileSet *pSet) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  // start compact
-  code = tsdbCompactFileSetStart(pCompactor, pSet);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  // do compact, end with a NULL row
-  SRowInfo *pRowInfo;
-  do {
-    code = tsdbCompactNextRow(pCompactor, &pRowInfo);
-    TSDB_CHECK_CODE(code, lino, _exit);
-
-    code = tsdbCompactWriteTableData(pCompactor, pRowInfo);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  } while (pRowInfo);
-
-  // end compact
-  code = tsdbCompactFileSetEnd(pCompactor);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino,
-              tstrerror(code), pCompactor->fid);
-    if (pCompactor->pWriter) tsdbDataFWriterClose(&pCompactor->pWriter, 0);
-    while ((pCompactor->pIter = pCompactor->iterList)) {
-      pCompactor->iterList = pCompactor->pIter->next;
-      tsdbCloseDataIter2(pCompactor->pIter);
-    }
-    if (pCompactor->pReader) tsdbDataFReaderClose(&pCompactor->pReader);
-  }
-  return code;
-}
-
-static void tsdbEndCompact(STsdbCompactor *pCompactor) {
-  // writer
-  tBlockDataDestroy(&pCompactor->sData);
-  tBlockDataDestroy(&pCompactor->bData);
-  taosArrayDestroy(pCompactor->aSttBlk);
-  tMapDataClear(&pCompactor->mDataBlk);
-  taosArrayDestroy(pCompactor->aBlockIdx);
-
-  // reader
-
-  // tombstone
-  taosArrayDestroy(pCompactor->aSkyLine);
-  taosArrayDestroy(pCompactor->aDelData);
-  taosArrayDestroy(pCompactor->aDelIdx);
-
-  // others
-  tDestroyTSchema(pCompactor->tbSkm.pTSchema);
-  tsdbFSDestroy(&pCompactor->fs);
-
-  tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->commitID);
-}
-
-static int32_t tsdbBeginCompact(STsdb *pTsdb, SCompactInfo *pInfo, STsdbCompactor *pCompactor) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  pCompactor->pTsdb = pTsdb;
-  pCompactor->commitID = pInfo->commitID;
-  pCompactor->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
-  pCompactor->maxRows = pTsdb->pVnode->config.tsdbCfg.maxRows;
-  pCompactor->minRows = pTsdb->pVnode->config.tsdbCfg.minRows;
-  pCompactor->fid = INT32_MIN;
-
-  code = tsdbFSCopy(pTsdb, &pCompactor->fs);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  /* tombstone */
-  if (pCompactor->fs.pDelFile) {
-    code = tsdbDelFReaderOpen(&pCompactor->pDelFReader, pCompactor->fs.pDelFile, pTsdb);
-    TSDB_CHECK_CODE(code, lino, _exit);
-
-    if ((pCompactor->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    if ((pCompactor->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    if ((pCompactor->aSkyLine = taosArrayInit(0, sizeof(TSDBKEY))) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    code = tsdbReadDelIdx(pCompactor->pDelFReader, pCompactor->aDelIdx);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  /* reader */
-
-  /* writer */
-  code = tBlockDataCreate(&pCompactor->bData);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  code = tBlockDataCreate(&pCompactor->sData);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, lino,
-              tstrerror(code), pCompactor->commitID);
-    tBlockDataDestroy(&pCompactor->sData);
-    tBlockDataDestroy(&pCompactor->bData);
-    if (pCompactor->fs.pDelFile) {
-      taosArrayDestroy(pCompactor->aSkyLine);
-      taosArrayDestroy(pCompactor->aDelData);
-      taosArrayDestroy(pCompactor->aDelIdx);
-      if (pCompactor->pDelFReader) tsdbDelFReaderClose(&pCompactor->pDelFReader);
-    }
-    tsdbFSDestroy(&pCompactor->fs);
-  } else {
-    tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, pCompactor->commitID);
-  }
-  return code;
-}
-
-int32_t tsdbCompact(STsdb *pTsdb, SCompactInfo *pInfo) {
-  int32_t code = 0;
-
-  STsdbCompactor *pCompactor = &(STsdbCompactor){0};
-
-  if ((code = tsdbBeginCompact(pTsdb, pInfo, pCompactor))) return code;
-
-  for (;;) {
-    SDFileSet *pSet = (SDFileSet *)taosArraySearch(pCompactor->fs.aDFileSet, &(SDFileSet){.fid = pCompactor->fid},
-                                                   tDFileSetCmprFn, TD_GT);
-    if (pSet == NULL) {
-      pCompactor->fid = INT32_MAX;
-      break;
-    }
-
-    if ((code = tsdbCompactFileSet(pCompactor, pSet))) goto _exit;
-  }
-
-  if ((code = tsdbFSUpsertDelFile(&pCompactor->fs, NULL))) goto _exit;
-
-_exit:
-  if (code) {
-    tsdbAbortCompact(pCompactor);
-  } else {
-    tsdbFSPrepareCommit(pTsdb, &pCompactor->fs);
-  }
-  tsdbEndCompact(pCompactor);
-  return code;
-}
-
-int32_t tsdbCommitCompact(STsdb *pTsdb) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  taosThreadRwlockWrlock(&pTsdb->rwLock);
-
-  code = tsdbFSCommit(pTsdb);
-  if (code) {
-    taosThreadRwlockUnlock(&pTsdb->rwLock);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  taosThreadRwlockUnlock(&pTsdb->rwLock);
-
-_exit:
-  if (code) {
-    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
-  } else {
-    tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__);
-  }
-  return code;
-}

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -404,7 +404,7 @@ void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) {
     tsdbDebug("try next last file block:%d from %d, trigger by uid:%" PRIu64 ", file index:%d, %s", pIter->iSttBlk,
               oldIndex, pIter->uid, pIter->iStt, idStr);
   } else {
-    tsdbDebug("no more last block qualified, uid:%" PRIu64 ", file index::%d, %s", pIter->uid, oldIndex, idStr);
+    tsdbDebug("no more last block qualified, uid:%" PRIu64 ", file index:%d, %s", pIter->uid, oldIndex, idStr);
   }
 }
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -44,6 +44,7 @@ typedef struct SBlockIndex {
 typedef struct STableBlockScanInfo {
   uint64_t  uid;
   TSKEY     lastKey;
+  TSKEY     lastKeyInStt;       // last accessed key in stt
   SMapData  mapData;            // block info (compressed)
   SArray*   pBlockList;         // block data index list, SArray<SBlockIndex>
   SIterInfo iter;               // mem buffer skip list iterator
@@ -192,7 +193,7 @@ static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbRe
 static int32_t  doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader,
                                         SRowMerger* pMerger);
 static int32_t  doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
-                                       SRowMerger* pMerger, SVersionRange* pVerRange);
+                                       SRowMerger* pMerger, SVersionRange* pVerRange, const char* id);
 static int32_t  doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
                                  STsdbReader* pReader);
 static int32_t  doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow,
@@ -314,11 +315,11 @@ static int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables)
   }
 
   if (pBuf->numOfTables > 0) {
-    STableBlockScanInfo **p = (STableBlockScanInfo**)taosArrayPop(pBuf->pData);
+    STableBlockScanInfo** p = (STableBlockScanInfo**)taosArrayPop(pBuf->pData);
     taosMemoryFree(*p);
     pBuf->numOfTables /= pBuf->numPerBucket;
   }
-  
+
   int32_t num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket;
   int32_t remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket;
   if (pBuf->pData == NULL) {
@@ -402,9 +403,11 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf
     if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
       int64_t skey = pTsdbReader->window.skey;
       pScanInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey;
+      pScanInfo->lastKeyInStt = skey;
     } else {
       int64_t ekey = pTsdbReader->window.ekey;
       pScanInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
+      pScanInfo->lastKeyInStt = ekey;
     }
 
     taosHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
@@ -1797,21 +1800,18 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
 
   while (1) {
     bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
-    if (!hasVal) {
+    if (!hasVal) {  // the next value will be the accessed key in stt
+      pScanInfo->lastKeyInStt += step;
       return false;
     }
 
     TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
     TSDBKEY k = TSDBROW_KEY(&row);
-    if (hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order, pVerRange)) {
+    pScanInfo->lastKeyInStt = k.ts;
-      pScanInfo->lastKey = k.ts;
+
-    } else {
+    if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order, pVerRange)) {
       // the qualifed ts may equal to k.ts, only a greater version one.
       // here we need to fallback one step.
-      if (pScanInfo->lastKey == k.ts) {
-        pScanInfo->lastKey -= step;
-      }
-
       return true;
     }
   }
@@ -1951,7 +1951,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
           return code;
         }
       }
-      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
+      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr);
     }
 
     if (minKey == k.ts) {
@@ -1999,7 +1999,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
           return code;
         }
       }
-      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
+      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr);
     }
 
     if (minKey == key) {
@@ -2052,7 +2052,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
 
       TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
       tsdbRowMerge(&merge, &fRow1);
-      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange);
+      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr);
 
       code = tsdbRowMergerGetRow(&merge, &pTSRow);
       if (code != TSDB_CODE_SUCCESS) {
@@ -2070,7 +2070,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
       return code;
     }
 
-    doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange);
+    doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr);
 
     // merge with block data if ts == key
     if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) {
@@ -2123,7 +2123,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
         TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
         tsdbRowMerge(&merge, &fRow1);
 
-        doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange);
+        doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange, pReader->idStr);
 
         code = tsdbRowMergerGetRow(&merge, &pTSRow);
         if (code != TSDB_CODE_SUCCESS) {
@@ -2232,7 +2232,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
         }
       }
 
-      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
+      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr);
     }
 
     if (minKey == ik.ts) {
@@ -2323,7 +2323,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
           return code;
         }
       }
-      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange);
+      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr);
     }
 
     if (minKey == key) {
@@ -2474,9 +2474,9 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
   int32_t     step = ASCENDING_TRAVERSE(pLBlockReader->order) ? 1 : -1;
   STimeWindow w = pLBlockReader->window;
   if (ASCENDING_TRAVERSE(pLBlockReader->order)) {
-    w.skey = pScanInfo->lastKey + step;
+    w.skey = pScanInfo->lastKeyInStt;
   } else {
-    w.ekey = pScanInfo->lastKey + step;
+    w.ekey = pScanInfo->lastKeyInStt;
   }
 
   tsdbDebug("init last block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey,
@@ -3514,6 +3514,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
                                          CHECK_FILEBLOCK_STATE* state) {
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
   SBlockData*         pBlockData = &pReader->status.fileBlockData;
+  bool                asc = ASCENDING_TRAVERSE(pReader->order);
 
   *state = CHECK_FILEBLOCK_QUIT;
   int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
@@ -3524,7 +3525,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
   if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) {
     pDumpInfo->rowIndex =
         doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, key, pMerger, &pReader->verRange, step);
-    if (pDumpInfo->rowIndex >= pDumpInfo->totalRows) {
+    if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) {
       *state = CHECK_FILEBLOCK_CONT;
     }
   }
@@ -3569,13 +3570,16 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
 }
 
 int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
-                               SRowMerger* pMerger, SVersionRange* pVerRange) {
+                               SRowMerger* pMerger, SVersionRange* pVerRange, const char* idStr) {
   while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) {
     int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
     if (next1 == ts) {
       TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
       tsdbRowMerge(pMerger, &fRow1);
     } else {
+      tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid,
+                pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt,
+                idStr);
       break;
     }
   }
@@ -3925,7 +3929,8 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n
     if (code) {
       return code;
     }
-    pReader->status.uidList.tableUidList = (uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num);
+    pReader->status.uidList.tableUidList =
+        (uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num);
   }
 
   taosHashClear(pReader->status.pTableMap);
@@ -3938,6 +3943,17 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n
     pInfo->uid = pList[i].uid;
     pUidList->tableUidList[i] = pList[i].uid;
 
+    // todo extract method
+    if (ASCENDING_TRAVERSE(pReader->order)) {
+      int64_t skey = pReader->window.skey;
+      pInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey;
+      pInfo->lastKeyInStt = skey;
+    } else {
+      int64_t ekey = pReader->window.ekey;
+      pInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
+      pInfo->lastKeyInStt = ekey;
+    }
+
     taosHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES);
   }
 
@@ -4723,11 +4739,10 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
 }
 
 static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) {
-  int32_t bucketIndex =  ((numOfRows - startRow) / bucketRange);
+  int32_t bucketIndex = ((numOfRows - startRow) / bucketRange);
   if (bucketIndex == numOfBucket) {
     bucketIndex -= 1;
   }
-
   return bucketIndex;
 }
 
@@ -4741,6 +4756,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
 
   // find the start data block in file
 
+  // find the start data block in file
   tsdbAcquireReader(pReader);
   if (pReader->suspended) {
     tsdbReaderResume(pReader);

source/dnode/vnode/src/vnd/vnodeCompact.c

@@ -1,120 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnd.h"
-
-extern int32_t tsdbCommitCompact(STsdb *pTsdb);
-
-static int32_t vnodeCompactTask(void *param) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  SCompactInfo *pInfo = (SCompactInfo *)param;
-  SVnode       *pVnode = pInfo->pVnode;
-
-  // do compact
-  code = tsdbCompact(pInfo->pVnode->pTsdb, pInfo);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  // end compact
-  char dir[TSDB_FILENAME_LEN] = {0};
-  if (pVnode->pTfs) {
-    snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
-  } else {
-    snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path);
-  }
-
-  vnodeCommitInfo(dir);
-
-  tsdbCommitCompact(pVnode->pTsdb);
-
-_exit:
-  tsem_post(&pInfo->pVnode->canCommit);
-  taosMemoryFree(pInfo);
-  return code;
-}
-static int32_t vnodePrepareCompact(SVnode *pVnode, SCompactInfo *pInfo) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  tsem_wait(&pVnode->canCommit);
-
-  pInfo->pVnode = pVnode;
-  pInfo->flag = 0;
-  pInfo->commitID = ++pVnode->state.commitID;
-
-  char       dir[TSDB_FILENAME_LEN] = {0};
-  SVnodeInfo info = {0};
-
-  if (pVnode->pTfs) {
-    snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
-  } else {
-    snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path);
-  }
-
-  if (vnodeLoadInfo(dir, &info) < 0) {
-    code = terrno;
-    goto _exit;
-  }
-
-  info.state.commitID = pInfo->commitID;
-
-  if (vnodeSaveInfo(dir, &info) < 0) {
-    code = terrno;
-    goto _exit;
-  }
-
-_exit:
-  if (code) {
-    vError("vgId:%d %s failed at line %d since %s, commit ID:%" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code),
-           pVnode->state.commitID);
-  } else {
-    vDebug("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pVnode), __func__, pVnode->state.commitID);
-  }
-  return code;
-}
-int32_t vnodeAsyncCompact(SVnode *pVnode) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  SCompactInfo *pInfo = taosMemoryCalloc(1, sizeof(*pInfo));
-  if (pInfo == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  vnodeAsyncCommit(pVnode);
-
-  code = vnodePrepareCompact(pVnode, pInfo);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  vnodeScheduleTask(vnodeCompactTask, pInfo);
-
-_exit:
-  if (code) {
-    vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
-    if (pInfo) taosMemoryFree(pInfo);
-  } else {
-    vInfo("vgId:%d %s done", TD_VID(pVnode), __func__);
-  }
-  return code;
-}
-
-int32_t vnodeSyncCompact(SVnode *pVnode) {
-  vnodeAsyncCompact(pVnode);
-  tsem_wait(&pVnode->canCommit);
-  tsem_post(&pVnode->canCommit);
-  return 0;
-}

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -211,6 +211,11 @@ static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) {
 
   SDecoder *pCoder = &(SDecoder){0};
 
+  if (taosHton64(((SSubmitReq2Msg *)pMsg->pCont)->version) != 1) {
+    code = TSDB_CODE_INVALID_MSG;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
   tDecoderInit(pCoder, (uint8_t *)pMsg->pCont + sizeof(SSubmitReq2Msg), pMsg->contLen - sizeof(SSubmitReq2Msg));
 
   if (tStartDecode(pCoder) < 0) {
@@ -599,16 +604,10 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq,
 
   vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp);
 
-// process
+  // process
-#if 0
-  code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp);
-  if (code) goto _exit;
-
-  code = smaDoRetention(pVnode->pSma, trimReq.timestamp);
-  if (code) goto _exit;
-#else
   vnodeAsyncRentention(pVnode, trimReq.timestamp);
-#endif
+  tsem_wait(&pVnode->canCommit);
+  tsem_post(&pVnode->canCommit);
 
 _exit:
   return code;
@@ -633,18 +632,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
     tqUpdateTbUidList(pVnode->pTq, tbUids, false);
   }
 
-#if 0
-  // process
-  ret = tsdbDoRetention(pVnode->pTsdb, ttlReq.timestamp);
-  if (ret) goto end;
-
-  ret = smaDoRetention(pVnode->pSma, ttlReq.timestamp);
-  if (ret) goto end;
-#else
   vnodeAsyncRentention(pVnode, ttlReq.timestamp);
-  tsem_wait(&pVnode->canCommit);
-  tsem_post(&pVnode->canCommit);
-#endif
 
 end:
   taosArrayDestroy(tbUids);
@@ -1228,6 +1216,49 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
     tDecoderClear(&dc);
   }
 
+  // scan
+  TSKEY now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
+  TSKEY minKey = now - tsTickPerMin[pVnode->config.tsdbCfg.precision] * pVnode->config.tsdbCfg.keep2;
+  TSKEY maxKey = tsMaxKeyByPrecision[pVnode->config.tsdbCfg.precision];
+  for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) {
+    SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i);
+
+    if (pSubmitTbData->pCreateTbReq && pSubmitTbData->pCreateTbReq->uid == 0) {
+      code = TSDB_CODE_INVALID_MSG;
+      goto _exit;
+    }
+
+    if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
+      if (TARRAY_SIZE(pSubmitTbData->aCol) <= 0) {
+        code = TSDB_CODE_INVALID_MSG;
+        goto _exit;
+      }
+
+      SColData *pColData = (SColData *)taosArrayGet(pSubmitTbData->aCol, 0);
+      TSKEY    *aKey = (TSKEY *)(pColData->pData);
+
+      for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) {
+        if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) {
+          code = TSDB_CODE_INVALID_MSG;
+          vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version);
+          goto _exit;
+        }
+      }
+
+    } else {
+      int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP);
+      SRow  **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP);
+
+      for (int32_t iRow = 0; iRow < nRow; ++iRow) {
+        if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) {
+          code = TSDB_CODE_INVALID_MSG;
+          vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version);
+          goto _exit;
+        }
+      }
+    }
+  }
+
   for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) {
     SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i);
 
@@ -1641,17 +1672,14 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p
   return TSDB_CODE_SUCCESS;
 }
 
+extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+
 static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  SCompactVnodeReq req = {0};
+  return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp);
-  if (tDeserializeSCompactVnodeReq(pReq, len, &req) != 0) {
+}
-    terrno = TSDB_CODE_INVALID_MSG;
-    return TSDB_CODE_INVALID_MSG;
-  }
-  vInfo("vgId:%d, compact msg will be processed, db:%s dbUid:%" PRId64 " compactStartTime:%" PRId64, TD_VID(pVnode),
-        req.db, req.dbUid, req.compactStartTime);
-
-  vnodeAsyncCompact(pVnode);
-  vnodeBegin(pVnode);
 
+#ifndef TD_ENTERPRISE
+int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
   return 0;
 }
+#endif

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -292,7 +292,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
 
     code = vnodePreProcessWriteMsg(pVnode, pMsg);
     if (code != 0) {
-      vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr());
+      vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, tstrerror(code));
       if (terrno != 0) code = terrno;
       vnodeHandleProposeError(pVnode, pMsg, code);
       rpcFreeCont(pMsg->pCont);

source/libs/command/src/command.c

@@ -248,13 +248,13 @@ static const char* cacheModelStr(int8_t cacheModel) {
   return TSDB_CACHE_MODEL_NONE_STR;
 }
 
-static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) {
+static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) {
   blockDataEnsureCapacity(pBlock, 1);
   pBlock->info.rows = 1;
 
   SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
   char             buf1[SHOW_CREATE_DB_RESULT_FIELD1_LEN] = {0};
-  STR_TO_VARSTR(buf1, dbFName);
+  STR_TO_VARSTR(buf1, dbName);
   colDataSetVal(pCol1, 0, buf1, false);
 
   SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
@@ -277,16 +277,20 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S
   }
 
   char* retentions = buildRetension(pCfg->pRetensions);
+  int32_t dbFNameLen = strlen(dbFName);
+  int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0;
 
   len += sprintf(
       buf2 + VARSTR_HEADER_SIZE,
       "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
       "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
-      "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d",
+      "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
-      dbFName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
+      "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " WAL_ROLL_PERIOD %d WAL_SEGMENT_SIZE %" PRId64,
+      dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
       pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows,  pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
       pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
-      1 == pCfg->numOfStables);
+      1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod,
+      pCfg->walRetentionSize, pCfg->walRollPeriod, pCfg->walSegmentSize);
 
   if (retentions) {
     len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);
@@ -404,7 +408,7 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT
   SSDataBlock* pBlock = NULL;
   int32_t      code = buildCreateDBResultDataBlock(&pBlock);
   if (TSDB_CODE_SUCCESS == code) {
-    setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg);
+    setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->dbFName, pStmt->pCfg);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp);

source/libs/executor/inc/executorimpl.h

@@ -881,6 +881,7 @@ SExprInfo*   createExpr(SNodeList* pNodeList, int32_t* numOfExprs);
 void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx,
                               SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo);
 void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset);
+void doClearBufferedBlocks(SStreamScanInfo* pInfo);
 
 #ifdef __cplusplus
 }

source/libs/executor/src/eventwindowoperator.c

@@ -29,7 +29,7 @@ typedef struct SEventWindowOperatorInfo {
   SWindowRowsSup     winSup;
   int32_t            tsSlotId;  // primary timestamp column slot id
   STimeWindowAggSupp twAggSup;
-  uint64_t           groupId;   // current group id, used to identify the data block from different groups
+  uint64_t           groupId;  // current group id, used to identify the data block from different groups
   SFilterInfo*       pStartCondInfo;
   SFilterInfo*       pEndCondInfo;
   bool               inWindow;
@@ -310,6 +310,7 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p
                                  pSup->rowEntryInfoOffset, pTaskInfo);
 
         pRes->info.rows += pInfo->pRow->numOfRows;
+        pInfo->pRow->numOfRows = 0;
 
         pInfo->inWindow = false;
         rowIndex += 1;

source/libs/executor/src/executil.c

@@ -957,9 +957,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
       STUidTagInfo* pInfo = taosArrayGet(pUidTagList, i);
       taosArrayPush(pUidList, &pInfo->uid);
     }
-
     terrno = 0;
-    goto end;
   } else {
     if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) {
       code = metaGetTableTagsByUids(metaHandle, pListInfo->suid, pUidTagList);

source/libs/executor/src/executor.c

@@ -93,6 +93,23 @@ static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) {
   return 0;
 }
 
+static void clearStreamBlock(SOperatorInfo* pOperator) {
+  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+    if (pOperator->numOfDownstream == 1) {
+      return clearStreamBlock(pOperator->pDownstream[0]);
+    }
+  } else {
+    SStreamScanInfo* pInfo = pOperator->info;
+    doClearBufferedBlocks(pInfo);
+  }
+}
+
+void resetTaskInfo(qTaskInfo_t tinfo) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  pTaskInfo->code = 0;
+  clearStreamBlock(pTaskInfo->pRoot);
+}
+
 static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
   if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     if (pOperator->numOfDownstream == 0) {
@@ -608,6 +625,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
   }
 
   if (isTaskKilled(pTaskInfo)) {
+    clearStreamBlock(pTaskInfo->pRoot);
     atomic_store_64(&pTaskInfo->owner, 0);
     qDebug("%s already killed, abort", GET_TASKID(pTaskInfo));
     return TSDB_CODE_SUCCESS;
@@ -1068,7 +1086,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       tsdbReaderClose(pTSInfo->base.dataReader);
       pTSInfo->base.dataReader = NULL;
       // let's seek to the next version in wal file
-      if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) {
+      if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) {
         return -1;
       }
     } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {

source/libs/executor/src/executorimpl.c

@@ -1197,16 +1197,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
     }
 
     if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
-      // expand the result datablock capacity
+      blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows);
-      if (pRow->numOfRows > pBlock->info.capacity) {
+      qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s",
-        blockDataEnsureCapacity(pBlock, pRow->numOfRows);
+             (pRow->numOfRows+pBlock->info.rows),
-        qDebug("datablock capacity not sufficient, expand to requried:%d, current capacity:%d, %s", pRow->numOfRows,
+             pBlock->info.capacity, GET_TASKID(pTaskInfo));
-               pBlock->info.capacity, GET_TASKID(pTaskInfo));
         // todo set the pOperator->resultInfo size
-      } else {
-        releaseBufPage(pBuf, page);
-        break;
-      }
     }
 
     pGroupResInfo->index += 1;

source/libs/executor/src/scanoperator.c

@@ -945,7 +945,8 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo*
   return pOperator;
 }
 
-static FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
+FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
+  qDebug("clear buff blocks:%d", (int32_t)taosArrayGetSize(pInfo->pBlockLists));
   taosArrayClear(pInfo->pBlockLists);
   pInfo->validBlockIndex = 0;
 }
@@ -1609,6 +1610,8 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
     if (pResult && pResult->info.rows > 0) {
       qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64, pResult->info.rows,
              pResult->info.window.skey, pResult->info.window.ekey);
+      qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows,
+             pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion);
       pTaskInfo->streamInfo.returned = 1;
       return pResult;
     } else {
@@ -1618,7 +1621,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
         pTSInfo->base.dataReader = NULL;
         tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
         qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
-        if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
+        if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
           tqOffsetResetToLog(&pTaskInfo->streamInfo.lastStatus, pTaskInfo->streamInfo.snapshotVer);
           return NULL;
         }
@@ -1632,8 +1635,12 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
     while (1) {
       SFetchRet ret = {0};
       if (tqNextBlock(pInfo->tqReader, &ret) < 0) {
-        qError("failed to get next log block since %s", terrstr());
+        // if the end is reached, terrno is 0
+        if (terrno != 0) {
+          qError("failed to get next log block since %s", terrstr());
+        }
       }
+
       if (ret.fetchType == FETCH_TYPE__DATA) {
         blockDataCleanup(pInfo->pRes);
         setBlockIntoRes(pInfo, &ret.data, true);
@@ -1651,8 +1658,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
       } else if (ret.fetchType == FETCH_TYPE__NONE ||
                  (ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) {
         pTaskInfo->streamInfo.lastStatus = ret.offset;
-        ASSERT(pTaskInfo->streamInfo.lastStatus.version >= pTaskInfo->streamInfo.prepareStatus.version);
-        ASSERT(pTaskInfo->streamInfo.lastStatus.version + 1 == pInfo->tqReader->pWalReader->curVersion);
         char formatBuf[80];
         tFormatOffset(formatBuf, 80, &ret.offset);
         qDebug("queue scan log return null, offset %s", formatBuf);
@@ -1660,16 +1665,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
         return NULL;
       }
     }
-#if 0
-    } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
-    if (pResult && pResult->info.rows > 0) {
-      qDebug("stream scan tsdb return %d rows", pResult->info.rows);
-      return pResult;
-    }
-    qDebug("stream scan tsdb return null");
-    return NULL;
-#endif
   } else {
     qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.prepareStatus.type);
     return NULL;
@@ -1919,7 +1914,9 @@ FETCH_NEXT_BLOCK:
         doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
         pInfo->pRes->info.dataLoad = 1;
         blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
-        return pInfo->pRes;
+        if (pInfo->pRes->info.rows > 0) {
+          return pInfo->pRes;
+        }
       } break;
       case STREAM_SCAN_FROM_DELETE_DATA: {
         generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);

source/libs/function/src/builtinsimpl.c

@@ -1940,6 +1940,7 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) {
   }
 
   if (pInfo->algo != APERCT_ALGO_TDIGEST) {
+    buildHistogramInfo(pInfo);
     qDebug("%s after merge, total:%" PRId64 ", numOfEntry:%d, %p", __FUNCTION__, pInfo->pHisto->numOfElems,
            pInfo->pHisto->numOfEntries, pInfo->pHisto);
   }
@@ -2461,6 +2462,9 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer
   int32_t numOfElems = 0;
 
   for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
+    if (colDataIsNull_s(pCol, i)) {
+      continue;
+    }
     char*          data = colDataGetData(pCol, i);
     SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data);
     int32_t        code = firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i);

source/libs/nodes/src/nodesUtilFuncs.c

@@ -963,8 +963,6 @@ void nodesDestroyNode(SNode* pNode) {
     case QUERY_NODE_SHOW_USERS_STMT:
     case QUERY_NODE_SHOW_LICENCES_STMT:
     case QUERY_NODE_SHOW_VGROUPS_STMT:
-    case QUERY_NODE_SHOW_DB_ALIVE_STMT:
-    case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT:
     case QUERY_NODE_SHOW_TOPICS_STMT:
     case QUERY_NODE_SHOW_CONSUMERS_STMT:
     case QUERY_NODE_SHOW_CONNECTIONS_STMT:

source/libs/parser/inc/parAst.h

@@ -151,7 +151,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo
 SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
 SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
 SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed);
-SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName);
+SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd);
 SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
 SNode* createAlterTableOptions(SAstCreateContext* pCxt);
 SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal);
@@ -215,6 +215,8 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool
                                 const SToken* pLibPath, SDataType dataType, int32_t bufSize, const SToken* pLanguage);
 SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName);
 SNode* createStreamOptions(SAstCreateContext* pCxt);
+SNode* setStreamOptions(SAstCreateContext* pCxt, SNode* pOptions, EStreamOptionsSetFlag setflag, SToken* pToken,
+                        SNode* pNode);
 SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable,
                               SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols);
 SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName);

source/libs/parser/inc/sql.y

@@ -167,7 +167,7 @@ cmd ::= USE db_name(A).
 cmd ::= ALTER DATABASE db_name(A) alter_db_options(B).                            { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
 cmd ::= FLUSH DATABASE db_name(A).                                                { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); }
 cmd ::= TRIM DATABASE db_name(A) speed_opt(B).                                    { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); }
-cmd ::= COMPACT DATABASE db_name(A).                                              { pCxt->pRootNode = createCompactStmt(pCxt, &A); }
+cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C).                      { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); }
 
 %type not_exists_opt                                                              { bool }
 %destructor not_exists_opt                                                        { }
@@ -259,6 +259,16 @@ retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C).
 speed_opt(A) ::= .                                                                { A = 0; }
 speed_opt(A) ::= MAX_SPEED NK_INTEGER(B).                                         { A = taosStr2Int32(B.z, NULL, 10); }
 
+start_opt(A) ::= .                                                                { A = NULL; }
+start_opt(A) ::= START WITH NK_INTEGER(B).                                        { A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
+start_opt(A) ::= START WITH NK_STRING(B).                                         { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
+start_opt(A) ::= START WITH TIMESTAMP NK_STRING(B).                               { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
+
+end_opt(A) ::= .                                                                  { A = NULL; }
+end_opt(A) ::= END WITH NK_INTEGER(B).                                            { A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
+end_opt(A) ::= END WITH NK_STRING(B).                                             { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
+end_opt(A) ::= END WITH TIMESTAMP NK_STRING(B).                                   { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
+
 /************************************************ create/drop table/stable ********************************************/
 cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
   NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E).                { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
@@ -567,14 +577,14 @@ tag_def_or_ref_opt(A) ::= tags_def(B).
 tag_def_or_ref_opt(A) ::= TAGS NK_LP col_name_list(B) NK_RP.                      { A = B; }
 
 stream_options(A) ::= .                                                           { A = createStreamOptions(pCxt); }
-stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE.                          { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_AT_ONCE; A = B; }
+stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE(C).                       { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, NULL); }
-stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE.                     { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; A = B; }
+stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE(C).                  { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, NULL); }
-stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY duration_literal(C).    { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; }
+stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY(C) duration_literal(D). { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, releaseRawExprNode(pCxt, D)); }
-stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C).            { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; }
+stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C).            { A = setStreamOptions(pCxt, B, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, C)); }
-stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C).             { ((SStreamOptions*)B)->ignoreExpired = taosStr2Int8(C.z, NULL, 10); A = B; }
+stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C).             { A = setStreamOptions(pCxt, B, SOPT_IGNORE_EXPIRED_SET, &C, NULL); }
-stream_options(A) ::= stream_options(B) FILL_HISTORY NK_INTEGER(C).               { ((SStreamOptions*)B)->fillHistory = taosStr2Int8(C.z, NULL, 10); A = B; }
+stream_options(A) ::= stream_options(B) FILL_HISTORY NK_INTEGER(C).               { A = setStreamOptions(pCxt, B, SOPT_FILL_HISTORY_SET, &C, NULL); }
-stream_options(A) ::= stream_options(B) DELETE_MARK duration_literal(C).          { ((SStreamOptions*)B)->pDeleteMark = releaseRawExprNode(pCxt, C); A = B; }
+stream_options(A) ::= stream_options(B) DELETE_MARK duration_literal(C).          { A = setStreamOptions(pCxt, B, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, C)); }
-stream_options(A) ::= stream_options(B) IGNORE UPDATE NK_INTEGER(C).              { ((SStreamOptions*)B)->ignoreUpdate = taosStr2Int8(C.z, NULL, 10); A = B; }
+stream_options(A) ::= stream_options(B) IGNORE UPDATE NK_INTEGER(C).              { A = setStreamOptions(pCxt, B, SOPT_IGNORE_UPDATE_SET, &C, NULL); }
 
 subtable_opt(A) ::= .                                                             { A = NULL; }
 subtable_opt(A) ::= SUBTABLE NK_LP expression(B) NK_RP.                           { A = releaseRawExprNode(pCxt, B); }

source/libs/parser/src/parAstCreater.c

@@ -1105,7 +1105,7 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t
   return (SNode*)pStmt;
 }
 
-SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName) {
+SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd) {
   CHECK_PARSER_STATUS(pCxt);
   if (!checkDbName(pCxt, pDbName, false)) {
     return NULL;
@@ -1113,6 +1113,8 @@ SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName) {
   SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)nodesMakeNode(QUERY_NODE_COMPACT_DATABASE_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
+  pStmt->pStart = pStart;
+  pStmt->pEnd = pEnd;
   return (SNode*)pStmt;
 }
 
@@ -1827,13 +1829,66 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) {
   CHECK_PARSER_STATUS(pCxt);
   SStreamOptions* pOptions = (SStreamOptions*)nodesMakeNode(QUERY_NODE_STREAM_OPTIONS);
   CHECK_OUT_OF_MEM(pOptions);
-  pOptions->triggerType = STREAM_TRIGGER_AT_ONCE;
+  pOptions->triggerType = STREAM_TRIGGER_WINDOW_CLOSE;
   pOptions->fillHistory = STREAM_DEFAULT_FILL_HISTORY;
   pOptions->ignoreExpired = STREAM_DEFAULT_IGNORE_EXPIRED;
   pOptions->ignoreUpdate = STREAM_DEFAULT_IGNORE_UPDATE;
   return (SNode*)pOptions;
 }
 
+static int8_t getTriggerType(uint32_t tokenType) {
+  switch (tokenType) {
+    case TK_AT_ONCE:
+      return STREAM_TRIGGER_AT_ONCE;
+    case TK_WINDOW_CLOSE:
+      return STREAM_TRIGGER_WINDOW_CLOSE;
+    case TK_MAX_DELAY:
+      return STREAM_TRIGGER_MAX_DELAY;
+    default:
+      break;
+  }
+  return STREAM_TRIGGER_WINDOW_CLOSE;
+}
+
+SNode* setStreamOptions(SAstCreateContext* pCxt, SNode* pOptions, EStreamOptionsSetFlag setflag, SToken* pToken,
+                        SNode* pNode) {
+  SStreamOptions* pStreamOptions = (SStreamOptions*)pOptions;
+  if (BIT_FLAG_TEST_MASK(setflag, pStreamOptions->setFlag)) {
+    pCxt->errCode =
+        generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "stream options each item is only set once");
+    return pOptions;
+  }
+
+  switch (setflag) {
+    case SOPT_TRIGGER_TYPE_SET:
+      pStreamOptions->triggerType = getTriggerType(pToken->type);
+      if (STREAM_TRIGGER_MAX_DELAY == pStreamOptions->triggerType) {
+        pStreamOptions->pDelay = pNode;
+      }
+      break;
+    case SOPT_WATERMARK_SET:
+      pStreamOptions->pWatermark = pNode;
+      break;
+    case SOPT_DELETE_MARK_SET:
+      pStreamOptions->pDeleteMark = pNode;
+      break;
+    case SOPT_FILL_HISTORY_SET:
+      pStreamOptions->fillHistory = taosStr2Int8(pToken->z, NULL, 10);
+      break;
+    case SOPT_IGNORE_EXPIRED_SET:
+      pStreamOptions->ignoreExpired = taosStr2Int8(pToken->z, NULL, 10);
+      break;
+    case SOPT_IGNORE_UPDATE_SET:
+      pStreamOptions->ignoreUpdate = taosStr2Int8(pToken->z, NULL, 10);
+      break;
+    default:
+      break;
+  }
+  BIT_FLAG_SET_MASK(pStreamOptions->setFlag, setflag);
+
+  return pOptions;
+}
+
 SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable,
                               SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) {
   CHECK_PARSER_STATUS(pCxt);

source/libs/parser/src/parAstParser.c

@@ -166,7 +166,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c
     code = reserveDnodeRequiredInCache(pCxt->pMetaCache);
   }
   if (TSDB_CODE_SUCCESS == code &&
-      (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES) || 0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) &&
+      (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES) ||
+       0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) &&
       QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) {
     code = collectMetaKeyFromInsTags(pCxt);
   }
@@ -605,6 +606,10 @@ static int32_t collectMetaKeyFromShowSubscriptions(SCollectMetaKeyCxt* pCxt, SSh
                                  pCxt->pMetaCache);
 }
 
+static int32_t collectMetaKeyFromCompactDatabase(SCollectMetaKeyCxt* pCxt, SCompactDatabaseStmt* pStmt) {
+  return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
+}
+
 static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
   pCxt->pStmt = pStmt;
   switch (nodeType(pStmt)) {
@@ -636,6 +641,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
       return collectMetaKeyFromExplain(pCxt, (SExplainStmt*)pStmt);
     case QUERY_NODE_DESCRIBE_STMT:
       return collectMetaKeyFromDescribe(pCxt, (SDescribeStmt*)pStmt);
+    case QUERY_NODE_COMPACT_DATABASE_STMT:
+      return collectMetaKeyFromCompactDatabase(pCxt, (SCompactDatabaseStmt*)pStmt);
     case QUERY_NODE_CREATE_STREAM_STMT:
       return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt);
     case QUERY_NODE_SHOW_DNODES_STMT:

source/libs/parser/src/parCalcConst.c

@@ -320,6 +320,79 @@ static int32_t calcConstInsert(SCalcConstContext* pCxt, SInsertStmt* pInsert) {
   return code;
 }
 
+static SNodeList* getChildProjection(SNode* pStmt) {
+  switch (nodeType(pStmt)) {
+    case QUERY_NODE_SELECT_STMT:
+      return ((SSelectStmt*)pStmt)->pProjectionList;
+    case QUERY_NODE_SET_OPERATOR:
+      return ((SSetOperator*)pStmt)->pProjectionList;
+    default:
+      break;
+  }
+  return NULL;
+}
+
+static void eraseSetOpChildProjection(SSetOperator* pSetOp, int32_t index) {
+  SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft);
+  nodesListErase(pLeftProjs, nodesListGetCell(pLeftProjs, index));
+  SNodeList* pRightProjs = getChildProjection(pSetOp->pRight);
+  nodesListErase(pRightProjs, nodesListGetCell(pRightProjs, index));
+}
+
+typedef struct SNotRefByOrderByCxt {
+  SColumnNode* pCol;
+  bool         hasThisCol;
+} SNotRefByOrderByCxt;
+
+static EDealRes notRefByOrderByImpl(SNode* pNode, void* pContext) {
+  if (QUERY_NODE_COLUMN == nodeType(pNode)) {
+    SNotRefByOrderByCxt* pCxt = (SNotRefByOrderByCxt*)pContext;
+    if (nodesEqualNode((SNode*)pCxt->pCol, pNode)) {
+      pCxt->hasThisCol = true;
+      return DEAL_RES_END;
+    }
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static bool notRefByOrderBy(SColumnNode* pCol, SNodeList* pOrderByList) {
+  SNotRefByOrderByCxt cxt = {.pCol = pCol, .hasThisCol = false};
+  nodesWalkExprs(pOrderByList, notRefByOrderByImpl, &cxt);
+  return !cxt.hasThisCol;
+}
+
+static int32_t calcConstSetOpProjections(SCalcConstContext* pCxt, SSetOperator* pSetOp, bool subquery) {
+  int32_t index = 0;
+  SNode*  pProj = NULL;
+  WHERE_EACH(pProj, pSetOp->pProjectionList) {
+    if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) && isUselessCol((SExprNode*)pProj)) {
+      ERASE_NODE(pSetOp->pProjectionList);
+      eraseSetOpChildProjection(pSetOp, index);
+      continue;
+    }
+    ++index;
+    WHERE_NEXT;
+  }
+  if (0 == LIST_LENGTH(pSetOp->pProjectionList)) {
+    return nodesListStrictAppend(pSetOp->pProjectionList, createConstantValue());
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t calcConstSetOperator(SCalcConstContext* pCxt, SSetOperator* pSetOp, bool subquery) {
+  int32_t code = calcConstSetOpProjections(pCxt, pSetOp, subquery);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = calcConstQuery(pCxt, pSetOp->pLeft, false);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = calcConstQuery(pCxt, pSetOp->pRight, false);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = calcConstList(pSetOp->pOrderByList);
+  }
+  return code;
+}
+
 static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery) {
   int32_t code = TSDB_CODE_SUCCESS;
   switch (nodeType(pStmt)) {
@@ -330,11 +403,7 @@ static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subque
       code = calcConstQuery(pCxt, ((SExplainStmt*)pStmt)->pQuery, subquery);
       break;
     case QUERY_NODE_SET_OPERATOR: {
-      SSetOperator* pSetOp = (SSetOperator*)pStmt;
+      code = calcConstSetOperator(pCxt, (SSetOperator*)pStmt, subquery);
-      code = calcConstQuery(pCxt, pSetOp->pLeft, false);
-      if (TSDB_CODE_SUCCESS == code) {
-        code = calcConstQuery(pCxt, pSetOp->pRight, false);
-      }
       break;
     }
     case QUERY_NODE_DELETE_STMT:

source/libs/parser/src/parTranslater.c

@@ -1211,7 +1211,8 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
   } else {
     res = translateNormalValue(pCxt, pVal, targetDt, strict);
   }
-  pVal->node.resType = targetDt;
+  pVal->node.resType.type = targetDt.type;
+  pVal->node.resType.bytes = targetDt.bytes;
   pVal->node.resType.scale = pVal->unit;
   pVal->translate = true;
   if (!strict && TSDB_DATA_TYPE_UBIGINT == pVal->node.resType.type && pVal->datum.u <= INT64_MAX) {
@@ -2499,6 +2500,12 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
       STempTableNode* pTempTable = (STempTableNode*)pTable;
       code = translateSubquery(pCxt, pTempTable->pSubquery);
       if (TSDB_CODE_SUCCESS == code) {
+        if (QUERY_NODE_SELECT_STMT == nodeType(pTempTable->pSubquery) && 
+           ((SSelectStmt*)pTempTable->pSubquery)->isEmptyResult &&
+           isSelectStmt(pCxt->pCurrStmt)) {
+          ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true;
+        }
+
         pTempTable->table.precision = getStmtPrecision(pTempTable->pSubquery);
         pTempTable->table.singleTable = stmtIsSingleTable(pTempTable->pSubquery);
         code = addNamespace(pCxt, pTempTable);
@@ -2796,6 +2803,10 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
   bool    other;
   int32_t code = translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other);
   if (TSDB_CODE_SUCCESS == code) {
+    if (0 == LIST_LENGTH(pSelect->pOrderByList)) {
+      NODES_DESTORY_LIST(pSelect->pOrderByList);
+      return TSDB_CODE_SUCCESS;
+    }
     if (!other) {
       return TSDB_CODE_SUCCESS;
     }
@@ -5621,12 +5632,36 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
   return refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta);
 }
 
+static int32_t translateCompactRange(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt, SCompactDbReq* pReq) {
+  SDbCfgInfo dbCfg = {0};
+  int32_t    code = getDBCfg(pCxt, pStmt->dbName, &dbCfg);
+  if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pStart) {
+    ((SValueNode*)pStmt->pStart)->node.resType.precision = dbCfg.precision;
+    ((SValueNode*)pStmt->pStart)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
+    code = doTranslateValue(pCxt, (SValueNode*)pStmt->pStart);
+  }
+  if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pEnd) {
+    ((SValueNode*)pStmt->pEnd)->node.resType.precision = dbCfg.precision;
+    ((SValueNode*)pStmt->pEnd)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
+    code = doTranslateValue(pCxt, (SValueNode*)pStmt->pEnd);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    pReq->timeRange.skey = NULL != pStmt->pStart ? ((SValueNode*)pStmt->pStart)->datum.i : INT64_MIN;
+    pReq->timeRange.ekey = NULL != pStmt->pEnd ? ((SValueNode*)pStmt->pEnd)->datum.i : INT64_MAX;
+  }
+  return code;
+}
+
 static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) {
   SCompactDbReq compactReq = {0};
   SName         name;
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
   tNameGetFullDbName(&name, compactReq.db);
-  return buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &compactReq);
+  int32_t code = translateCompactRange(pCxt, pStmt, &compactReq);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &compactReq);
+  }
+  return code;
 }
 
 static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) {
@@ -6379,15 +6414,15 @@ static int32_t translateDropFunction(STranslateContext* pCxt, SDropFunctionStmt*
 
 static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
   SAlterUserReq req = {0};
-  if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) ||
+  if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) ||
-      (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) &&
+      (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) &&
-       PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) {
+       BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) {
     req.alterType = TSDB_ALTER_USER_ADD_ALL_DB;
-  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) {
+  } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) {
     req.alterType = TSDB_ALTER_USER_ADD_READ_DB;
-  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
+  } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
     req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB;
-  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
+  } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
     req.alterType = TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC;
   }
   strcpy(req.user, pStmt->userName);
@@ -6397,15 +6432,15 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
 
 static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
   SAlterUserReq req = {0};
-  if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) ||
+  if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) ||
-      (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) &&
+      (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) &&
-       PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) {
+       BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) {
     req.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB;
-  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) {
+  } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) {
     req.alterType = TSDB_ALTER_USER_REMOVE_READ_DB;
-  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
+  } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
     req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB;
-  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
+  } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
     req.alterType = TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC;
   }
   strcpy(req.user, pStmt->userName);
@@ -6477,6 +6512,11 @@ static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateD
   if (NULL == pStmt->pCfg) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+
+  SName name;
+  tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
+  tNameGetFullDbName(&name, pStmt->dbFName);
+
   return getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pCfg);
 }
 

source/libs/parser/test/mockCatalog.cpp

@@ -226,6 +226,7 @@ void generateDatabases(MockCatalogService* mcs) {
   generateTestTables(g_mockCatalogService.get(), "cache_db");
   generateTestStables(g_mockCatalogService.get(), "cache_db");
   mcs->createDatabase("rollup_db", true);
+  mcs->createDatabase("testus", false, 0, TSDB_TIME_PRECISION_NANO);
 }
 
 }  // namespace
@@ -252,7 +253,8 @@ int32_t __catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableNam
   return code;
 }
 
-int32_t __catalogGetCachedTableVgMeta(SCatalog* pCtg, const SName* pTableName,          SVgroupInfo* pVgroup, STableMeta** pTableMeta) {
+int32_t __catalogGetCachedTableVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup,
+                                      STableMeta** pTableMeta) {
   int32_t code = g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta, true);
   if (code) return code;
   code = g_mockCatalogService->catalogGetTableHashVgroup(pTableName, pVgroup, true);

source/libs/parser/test/mockCatalogService.cpp

@@ -346,12 +346,13 @@ class MockCatalogServiceImpl {
     dnode_.insert(std::make_pair(dnodeId, epSet));
   }
 
-  void createDatabase(const string& db, bool rollup, int8_t cacheLast) {
+  void createDatabase(const string& db, bool rollup, int8_t cacheLast, int8_t precision) {
     SDbCfgInfo cfg = {0};
     if (rollup) {
       cfg.pRetensions = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SRetention));
     }
     cfg.cacheLast = cacheLast;
+    cfg.precision = precision;
     dbCfg_.insert(std::make_pair(db, cfg));
   }
 
@@ -681,8 +682,8 @@ void MockCatalogService::createDnode(int32_t dnodeId, const string& host, int16_
   impl_->createDnode(dnodeId, host, port);
 }
 
-void MockCatalogService::createDatabase(const string& db, bool rollup, int8_t cacheLast) {
+void MockCatalogService::createDatabase(const string& db, bool rollup, int8_t cacheLast, int8_t precision) {
-  impl_->createDatabase(db, rollup, cacheLast);
+  impl_->createDatabase(db, rollup, cacheLast, precision);
 }
 
 int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta,

source/libs/parser/test/mockCatalogService.h

@@ -65,7 +65,8 @@ class MockCatalogService {
   void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize);
   void createSmaIndex(const SMCreateSmaReq* pReq);
   void createDnode(int32_t dnodeId, const std::string& host, int16_t port);
-  void createDatabase(const std::string& db, bool rollup = false, int8_t cacheLast = 0);
+  void createDatabase(const std::string& db, bool rollup = false, int8_t cacheLast = 0,
+                      int8_t precision = TSDB_TIME_PRECISION_MILLI);
 
   int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, bool onlyCache = false) const;
   int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, bool onlyCache = false) const;

source/libs/parser/test/parInitialCTest.cpp

@@ -24,12 +24,16 @@ namespace ParserTest {
 class ParserInitialCTest : public ParserDdlTest {};
 
 /*
- * COMPACT DATABASE db_name
+ * COMPACT DATABASE db_name [START WITH start_time] [END WITH END_time]
  */
 TEST_F(ParserInitialCTest, compact) {
   SCompactDbReq expect = {0};
 
-  auto setCompactDbReq = [&](const char* pDb) { snprintf(expect.db, sizeof(expect.db), "0.%s", pDb); };
+  auto setCompactDbReq = [&](const char* pDb, int64_t start = INT64_MIN, int64_t end = INT64_MAX) {
+    snprintf(expect.db, sizeof(expect.db), "0.%s", pDb);
+    expect.timeRange.skey = start;
+    expect.timeRange.ekey = end;
+  };
 
   setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
     ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_COMPACT_DATABASE_STMT);
@@ -37,10 +41,21 @@ TEST_F(ParserInitialCTest, compact) {
     SCompactDbReq req = {0};
     ASSERT_EQ(tDeserializeSCompactDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
     ASSERT_EQ(std::string(req.db), std::string(expect.db));
+    ASSERT_EQ(req.timeRange.skey, expect.timeRange.skey);
+    ASSERT_EQ(req.timeRange.ekey, expect.timeRange.ekey);
   });
 
-  setCompactDbReq("wxy_db");
+  setCompactDbReq("test");
-  run("COMPACT DATABASE wxy_db");
+  run("COMPACT DATABASE test");
+
+  setCompactDbReq("test", 1678168883000, 1678255283000);
+  run("COMPACT DATABASE test START WITH '2023-03-07 14:01:23' END WITH '2023-03-08 14:01:23'");
+
+  setCompactDbReq("testus", 1673071283000000000);
+  run("COMPACT DATABASE testus START WITH TIMESTAMP '2023-01-07 14:01:23'");
+
+  setCompactDbReq("testus", INT64_MIN, 1675749683000000000);
+  run("COMPACT DATABASE testus END WITH 1675749683000000000");
 }
 
 /*
@@ -797,16 +812,10 @@ TEST_F(ParserInitialCTest, createStream) {
     snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb);
     expect.igExists = igExists;
     expect.sql = taosStrdup(pSql);
-    expect.createStb = STREAM_CREATE_STABLE_TRUE;
-    expect.triggerType = STREAM_TRIGGER_AT_ONCE;
-    expect.maxDelay = 0;
-    expect.watermark = 0;
-    expect.fillHistory = STREAM_DEFAULT_FILL_HISTORY;
-    expect.igExpired = STREAM_DEFAULT_IGNORE_EXPIRED;
   };
 
   auto setStreamOptions =
-      [&](int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t triggerType = STREAM_TRIGGER_AT_ONCE,
+      [&](int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t triggerType = STREAM_TRIGGER_WINDOW_CLOSE,
           int64_t maxDelay = 0, int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED,
           int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY, int8_t igUpdate = STREAM_DEFAULT_IGNORE_UPDATE) {
         expect.createStb = createStb;
@@ -868,6 +877,7 @@ TEST_F(ParserInitialCTest, createStream) {
   });
 
   setCreateStreamReq("s1", "test", "create stream s1 into st3 as select count(*) from t1 interval(10s)", "st3");
+  setStreamOptions();
   run("CREATE STREAM s1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)");
   clearCreateStreamReq();
 
@@ -888,6 +898,7 @@ TEST_F(ParserInitialCTest, createStream) {
                      "st3");
   addTag("tname", TSDB_DATA_TYPE_VARCHAR, 10 + VARSTR_HEADER_SIZE);
   addTag("id", TSDB_DATA_TYPE_INT);
+  setStreamOptions();
   run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) "
       "AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 PARTITION BY TBNAME tname, tag1 id INTERVAL(10S)");
   clearCreateStreamReq();

source/libs/parser/test/parTestUtil.cpp

@@ -65,15 +65,16 @@ int32_t getLogLevel() { return g_logLevel; }
 
 class ParserTestBaseImpl {
  public:
-  ParserTestBaseImpl(ParserTestBase* pBase) : pBase_(pBase), sqlNo_(0), sqlNum_(0) {}
+  ParserTestBaseImpl(ParserTestBase* pBase) : pBase_(pBase), sqlNo_(0), sqlNum_(0) {
+    caseEnv_.numOfSkipSql_ = g_skipSql;
+    caseEnv_.numOfLimitSql_ = g_limitSql;
+  }
 
   void login(const std::string& user) { caseEnv_.user_ = user; }
 
   void useDb(const string& acctId, const string& db) {
     caseEnv_.acctId_ = acctId;
     caseEnv_.db_ = db;
-    caseEnv_.numOfSkipSql_ = g_skipSql;
-    caseEnv_.numOfLimitSql_ = g_limitSql;
   }
 
   void run(const string& sql, int32_t expect, ParserStage checkStage) {

source/libs/scalar/src/sclvector.c

@@ -915,6 +915,7 @@ int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_
 
   SDataType t = {.type = type};
   t.bytes = IS_VAR_DATA_TYPE(t.type)? input->columnData->info.bytes:tDataTypes[type].bytes;
+  t.precision = input->columnData->info.precision;
 
   int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output);
   if (code != TSDB_CODE_SUCCESS) {

source/libs/stream/src/streamDispatch.c

@@ -278,6 +278,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov
   msg.contLen = tlen + sizeof(SMsgHead);
   msg.pCont = buf;
   msg.msgType = TDMT_STREAM_RECOVER_FINISH;
+  msg.info.noResp = 1;
 
   tmsgSendReq(pEpSet, &msg);
 
@@ -522,4 +523,3 @@ FREE:
   taosFreeQitem(pBlock);
   return code;
 }
-

source/libs/stream/src/streamExec.c

@@ -20,6 +20,11 @@
 static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
   int32_t code;
   void*   exec = pTask->exec.executor;
+  while(atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
+    qError("stream task wait for the end of fill history");
+    taosMsleep(2);
+    continue;
+  }
 
   // set input
   const SStreamQueueItem* pItem = (const SStreamQueueItem*)data;
@@ -58,6 +63,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
     SSDataBlock* output = NULL;
     uint64_t     ts = 0;
     if ((code = qExecTask(exec, &output, &ts)) < 0) {
+      if (code == TSDB_CODE_QRY_IN_EXEC) {
+        resetTaskInfo(exec);
+      }
       /*ASSERT(false);*/
       qError("unexpected stream execution, stream %" PRId64 " task: %d,  since %s", pTask->streamId, pTask->taskId,
              terrstr());
@@ -121,8 +129,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
       SSDataBlock* output = NULL;
       uint64_t     ts = 0;
       if (qExecTask(exec, &output, &ts) < 0) {
-        taosArrayDestroy(pRes);
+        continue;
-        return -1;
       }
       if (output == NULL) {
         if (qStreamRecoverScanFinished(exec)) {

source/libs/stream/src/streamRecover.c

@@ -168,7 +168,7 @@ int32_t streamRestoreParam(SStreamTask* pTask) {
   return qStreamRestoreParam(exec);
 }
 int32_t streamSetStatusNormal(SStreamTask* pTask) {
-  pTask->taskStatus = TASK_STATUS__NORMAL;
+  atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
   return 0;
 }
 

source/libs/sync/inc/syncReplication.h

@@ -51,7 +51,7 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode);
 int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* pDestId, SRpcMsg* pMsg);
 
 int32_t syncNodeReplicate(SSyncNode* pSyncNode);
-int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapshot);
+int32_t syncNodeReplicateReset(SSyncNode* pSyncNode, SRaftId* pDestId);
 int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode);
 
 int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);

source/libs/sync/src/syncMain.c

@@ -285,7 +285,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) {
 
   if (syncNodeIsMnode(pSyncNode)) {
     // mnode
-    logRetention = SYNC_MNODE_LOG_RETENTION;
+    logRetention = tsMndLogRetention;
   } else {
     // vnode
     if (pSyncNode->replicaNum > 1) {
@@ -301,7 +301,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) {
       syncNodeRelease(pSyncNode);
       return 0;
     }
-    logRetention = TMAX(logRetention, lastApplyIndex - pSyncNode->minMatchIndex);
+    logRetention = TMAX(logRetention, lastApplyIndex - pSyncNode->minMatchIndex + logRetention);
   }
 
 _DEL_WAL: