Merge branch 'refact/submit_req' of github.com:taosdata/TDengine into refact/submit_req

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG cf30c86
+        GIT_TAG d5df76d
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/zh/02-intro.md

@@ -69,7 +69,7 @@ TDengine 的主要功能如下：
 
 - **[分析能力](https://www.taosdata.com/tdengine/easy_data_analytics)**：通过超级表、存储计算分离、分区分片、预计算和其它技术，TDengine 能够高效地浏览、格式化和访问数据。
 
-- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 19k，且拥有一个活跃的开发者社区。
+- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 20k，且拥有一个活跃的开发者社区。
 
 采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。表现在几个方面：
 

docs/zh/27-train-faq/01-faq.md

@@ -201,3 +201,45 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端
 OOM 是操作系统的保护机制，当操作系统内存(包括 SWAP )不足时，会杀掉某些进程，从而保证操作系统的稳定运行。通常内存不足主要是如下两个原因导致，一是剩余内存小于 vm.min_free_kbytes ；二是程序请求的内存大于剩余内存。还有一种情况是内存充足但程序占用了特殊的内存地址，也会触发 OOM 。
 
 TDengine 会预先为每个 VNode 分配好内存，每个 Database 的 VNode 个数受 建库时的vgroups参数影响，每个 VNode 占用的内存大小受 buffer参数 影响。要防止 OOM，需要在项目建设之初合理规划内存，并合理设置 SWAP ，除此之外查询过量的数据也有可能导致内存暴涨，这取决于具体的查询语句。TDengine 企业版对内存管理做了优化，采用了新的内存分配器，对稳定性有更高要求的用户可以考虑选择企业版。
+
+### 19. 在macOS上遇到Too many open files怎么办？
+
+taosd日志文件报错Too many open file，是由于taosd打开文件数超过系统设置的上限所致。
+解决方案如下：
+1. 新建文件 /Library/LaunchDaemons/limit.maxfiles.plist，写入以下内容(以下示例将limit和maxfiles改为10万，可按需修改)：
+```
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
+"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+<key>Label</key>
+  <string>limit.maxfiles</string>
+<key>ProgramArguments</key>
+<array>
+  <string>launchctl</string>
+  <string>limit</string>
+  <string>maxfiles</string>
+  <string>100000</string>
+  <string>100000</string>
+</array>
+<key>RunAtLoad</key>
+  <true/>
+<key>ServiceIPC</key>
+  <false/>
+</dict>
+</plist>
+```
+2. 修改文件权限
+```
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
+sudo chmod 644 /Library/LaunchDaemons/limit.maxfiles.plist
+```
+3. 加载 plist 文件 (或重启系统后生效。launchd在启动时会自动加载该目录的 plist)
+```
+sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
+```
+4.确认更改后的限制
+```
+launchctl limit maxfiles
+```

include/common/systable.h

@@ -47,6 +47,7 @@ extern "C" {
 #define TSDB_INS_TABLE_TOPICS            "ins_topics"
 #define TSDB_INS_TABLE_STREAMS           "ins_streams"
 #define TSDB_INS_TABLE_STREAM_TASKS      "ins_stream_tasks"
+#define TSDB_INS_TABLE_USER_PRIVILEGES   "ins_user_privileges"
 
 #define TSDB_PERFORMANCE_SCHEMA_DB   "performance_schema"
 #define TSDB_PERFS_TABLE_SMAS        "perf_smas"

include/common/tdataformat.h

@@ -94,7 +94,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
 int32_t  tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);
 void     tRowIterClose(SRowIter **ppIter);
 SColVal *tRowIterNext(SRowIter *pIter);
-int32_t  tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SArray *aColData, int32_t nColData);
+int32_t  tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData);
 
 // STag ================================
 int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag);
@@ -111,6 +111,7 @@ void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remov
 int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf);
 
 // SColData ================================
+typedef void *(*xMallocFn)(void *, int32_t);
 void    tColDataDestroy(void *ph);
 void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
 void    tColDataClear(SColData *pColData);
@@ -118,7 +119,7 @@ void    tColDataDeepClear(SColData *pColData);
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
 void    tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
 uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
+int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg);
 extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull);
 
 // for stmt bind

include/common/tglobal.h

@@ -129,6 +129,7 @@ extern char tsUdfdLdLibPath[];
 extern char tsSmlChildTableName[];
 extern char tsSmlTagName[];
 extern bool tsSmlDataFormat;
+extern int32_t tsSmlBatchSize;
 
 // wal
 extern int64_t tsWalFsyncDataSizeLimit;

include/common/tmsg.h

@@ -151,6 +151,8 @@ typedef enum _mgmt_table {
 #define TSDB_ALTER_USER_REMOVE_ALL_DB          0x8
 #define TSDB_ALTER_USER_ENABLE                 0x9
 #define TSDB_ALTER_USER_SYSINFO                0xA
+#define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC    0xB
+#define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC
 
 #define TSDB_ALTER_USER_PRIVILEGES 0x2
 
@@ -618,7 +620,7 @@ typedef struct {
   int8_t enable;
   char   user[TSDB_USER_LEN];
   char   pass[TSDB_USET_PASSWORD_LEN];
-  char   dbname[TSDB_DB_FNAME_LEN];
+  char   objname[TSDB_DB_FNAME_LEN];  // db or topic
 } SAlterUserReq;
 
 int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
@@ -1057,6 +1059,7 @@ typedef struct {
   int32_t vgId;
   int8_t  syncState;
   int8_t  syncRestore;
+  int8_t  syncCanRead;
   int64_t cacheUsage;
   int64_t numOfTables;
   int64_t numOfTimeSeries;

include/common/ttokendef.h

@@ -58,282 +58,284 @@
 #define TK_TO                   40
 #define TK_REVOKE               41
 #define TK_FROM                 42
-#define TK_NK_COMMA             43
+#define TK_SUBSCRIBE            43
-#define TK_READ                 44
+#define TK_NK_COMMA             44
-#define TK_WRITE                45
+#define TK_READ                 45
-#define TK_NK_DOT               46
+#define TK_WRITE                46
-#define TK_DNODE                47
+#define TK_NK_DOT               47
-#define TK_PORT                 48
+#define TK_DNODE                48
-#define TK_DNODES               49
+#define TK_PORT                 49
-#define TK_NK_IPTOKEN           50
+#define TK_DNODES               50
-#define TK_FORCE                51
+#define TK_NK_IPTOKEN           51
-#define TK_LOCAL                52
+#define TK_FORCE                52
-#define TK_QNODE                53
+#define TK_LOCAL                53
-#define TK_BNODE                54
+#define TK_QNODE                54
-#define TK_SNODE                55
+#define TK_BNODE                55
-#define TK_MNODE                56
+#define TK_SNODE                56
-#define TK_DATABASE             57
+#define TK_MNODE                57
-#define TK_USE                  58
+#define TK_DATABASE             58
-#define TK_FLUSH                59
+#define TK_USE                  59
-#define TK_TRIM                 60
+#define TK_FLUSH                60
-#define TK_IF                   61
+#define TK_TRIM                 61
-#define TK_NOT                  62
+#define TK_IF                   62
-#define TK_EXISTS               63
+#define TK_NOT                  63
-#define TK_BUFFER               64
+#define TK_EXISTS               64
-#define TK_CACHEMODEL           65
+#define TK_BUFFER               65
-#define TK_CACHESIZE            66
+#define TK_CACHEMODEL           66
-#define TK_COMP                 67
+#define TK_CACHESIZE            67
-#define TK_DURATION             68
+#define TK_COMP                 68
-#define TK_NK_VARIABLE          69
+#define TK_DURATION             69
-#define TK_MAXROWS              70
+#define TK_NK_VARIABLE          70
-#define TK_MINROWS              71
+#define TK_MAXROWS              71
-#define TK_KEEP                 72
+#define TK_MINROWS              72
-#define TK_PAGES                73
+#define TK_KEEP                 73
-#define TK_PAGESIZE             74
+#define TK_PAGES                74
-#define TK_TSDB_PAGESIZE        75
+#define TK_PAGESIZE             75
-#define TK_PRECISION            76
+#define TK_TSDB_PAGESIZE        76
-#define TK_REPLICA              77
+#define TK_PRECISION            77
-#define TK_STRICT               78
+#define TK_REPLICA              78
-#define TK_VGROUPS              79
+#define TK_STRICT               79
-#define TK_SINGLE_STABLE        80
+#define TK_VGROUPS              80
-#define TK_RETENTIONS           81
+#define TK_SINGLE_STABLE        81
-#define TK_SCHEMALESS           82
+#define TK_RETENTIONS           82
-#define TK_WAL_LEVEL            83
+#define TK_SCHEMALESS           83
-#define TK_WAL_FSYNC_PERIOD     84
+#define TK_WAL_LEVEL            84
-#define TK_WAL_RETENTION_PERIOD 85
+#define TK_WAL_FSYNC_PERIOD     85
-#define TK_WAL_RETENTION_SIZE   86
+#define TK_WAL_RETENTION_PERIOD 86
-#define TK_WAL_ROLL_PERIOD      87
+#define TK_WAL_RETENTION_SIZE   87
-#define TK_WAL_SEGMENT_SIZE     88
+#define TK_WAL_ROLL_PERIOD      88
-#define TK_STT_TRIGGER          89
+#define TK_WAL_SEGMENT_SIZE     89
-#define TK_TABLE_PREFIX         90
+#define TK_STT_TRIGGER          90
-#define TK_TABLE_SUFFIX         91
+#define TK_TABLE_PREFIX         91
-#define TK_NK_COLON             92
+#define TK_TABLE_SUFFIX         92
-#define TK_MAX_SPEED            93
+#define TK_NK_COLON             93
-#define TK_TABLE                94
+#define TK_MAX_SPEED            94
-#define TK_NK_LP                95
+#define TK_TABLE                95
-#define TK_NK_RP                96
+#define TK_NK_LP                96
-#define TK_STABLE               97
+#define TK_NK_RP                97
-#define TK_ADD                  98
+#define TK_STABLE               98
-#define TK_COLUMN               99
+#define TK_ADD                  99
-#define TK_MODIFY               100
+#define TK_COLUMN               100
-#define TK_RENAME               101
+#define TK_MODIFY               101
-#define TK_TAG                  102
+#define TK_RENAME               102
-#define TK_SET                  103
+#define TK_TAG                  103
-#define TK_NK_EQ                104
+#define TK_SET                  104
-#define TK_USING                105
+#define TK_NK_EQ                105
-#define TK_TAGS                 106
+#define TK_USING                106
-#define TK_COMMENT              107
+#define TK_TAGS                 107
-#define TK_BOOL                 108
+#define TK_COMMENT              108
-#define TK_TINYINT              109
+#define TK_BOOL                 109
-#define TK_SMALLINT             110
+#define TK_TINYINT              110
-#define TK_INT                  111
+#define TK_SMALLINT             111
-#define TK_INTEGER              112
+#define TK_INT                  112
-#define TK_BIGINT               113
+#define TK_INTEGER              113
-#define TK_FLOAT                114
+#define TK_BIGINT               114
-#define TK_DOUBLE               115
+#define TK_FLOAT                115
-#define TK_BINARY               116
+#define TK_DOUBLE               116
-#define TK_TIMESTAMP            117
+#define TK_BINARY               117
-#define TK_NCHAR                118
+#define TK_TIMESTAMP            118
-#define TK_UNSIGNED             119
+#define TK_NCHAR                119
-#define TK_JSON                 120
+#define TK_UNSIGNED             120
-#define TK_VARCHAR              121
+#define TK_JSON                 121
-#define TK_MEDIUMBLOB           122
+#define TK_VARCHAR              122
-#define TK_BLOB                 123
+#define TK_MEDIUMBLOB           123
-#define TK_VARBINARY            124
+#define TK_BLOB                 124
-#define TK_DECIMAL              125
+#define TK_VARBINARY            125
-#define TK_MAX_DELAY            126
+#define TK_DECIMAL              126
-#define TK_WATERMARK            127
+#define TK_MAX_DELAY            127
-#define TK_ROLLUP               128
+#define TK_WATERMARK            128
-#define TK_TTL                  129
+#define TK_ROLLUP               129
-#define TK_SMA                  130
+#define TK_TTL                  130
-#define TK_FIRST                131
+#define TK_SMA                  131
-#define TK_LAST                 132
+#define TK_FIRST                132
-#define TK_SHOW                 133
+#define TK_LAST                 133
-#define TK_DATABASES            134
+#define TK_SHOW                 134
-#define TK_TABLES               135
+#define TK_PRIVILEGES           135
-#define TK_STABLES              136
+#define TK_DATABASES            136
-#define TK_MNODES               137
+#define TK_TABLES               137
-#define TK_QNODES               138
+#define TK_STABLES              138
-#define TK_FUNCTIONS            139
+#define TK_MNODES               139
-#define TK_INDEXES              140
+#define TK_QNODES               140
-#define TK_ACCOUNTS             141
+#define TK_FUNCTIONS            141
-#define TK_APPS                 142
+#define TK_INDEXES              142
-#define TK_CONNECTIONS          143
+#define TK_ACCOUNTS             143
-#define TK_LICENCES             144
+#define TK_APPS                 144
-#define TK_GRANTS               145
+#define TK_CONNECTIONS          145
-#define TK_QUERIES              146
+#define TK_LICENCES             146
-#define TK_SCORES               147
+#define TK_GRANTS               147
-#define TK_TOPICS               148
+#define TK_QUERIES              148
-#define TK_VARIABLES            149
+#define TK_SCORES               149
-#define TK_CLUSTER              150
+#define TK_TOPICS               150
-#define TK_BNODES               151
+#define TK_VARIABLES            151
-#define TK_SNODES               152
+#define TK_CLUSTER              152
-#define TK_TRANSACTIONS         153
+#define TK_BNODES               153
-#define TK_DISTRIBUTED          154
+#define TK_SNODES               154
-#define TK_CONSUMERS            155
+#define TK_TRANSACTIONS         155
-#define TK_SUBSCRIPTIONS        156
+#define TK_DISTRIBUTED          156
-#define TK_VNODES               157
+#define TK_CONSUMERS            157
-#define TK_LIKE                 158
+#define TK_SUBSCRIPTIONS        158
-#define TK_TBNAME               159
+#define TK_VNODES               159
-#define TK_QTAGS                160
+#define TK_LIKE                 160
-#define TK_AS                   161
+#define TK_TBNAME               161
-#define TK_INDEX                162
+#define TK_QTAGS                162
-#define TK_FUNCTION             163
+#define TK_AS                   163
-#define TK_INTERVAL             164
+#define TK_INDEX                164
-#define TK_TOPIC                165
+#define TK_FUNCTION             165
-#define TK_WITH                 166
+#define TK_INTERVAL             166
-#define TK_META                 167
+#define TK_TOPIC                167
-#define TK_CONSUMER             168
+#define TK_WITH                 168
-#define TK_GROUP                169
+#define TK_META                 169
-#define TK_DESC                 170
+#define TK_CONSUMER             170
-#define TK_DESCRIBE             171
+#define TK_GROUP                171
-#define TK_RESET                172
+#define TK_DESC                 172
-#define TK_QUERY                173
+#define TK_DESCRIBE             173
-#define TK_CACHE                174
+#define TK_RESET                174
-#define TK_EXPLAIN              175
+#define TK_QUERY                175
-#define TK_ANALYZE              176
+#define TK_CACHE                176
-#define TK_VERBOSE              177
+#define TK_EXPLAIN              177
-#define TK_NK_BOOL              178
+#define TK_ANALYZE              178
-#define TK_RATIO                179
+#define TK_VERBOSE              179
-#define TK_NK_FLOAT             180
+#define TK_NK_BOOL              180
-#define TK_OUTPUTTYPE           181
+#define TK_RATIO                181
-#define TK_AGGREGATE            182
+#define TK_NK_FLOAT             182
-#define TK_BUFSIZE              183
+#define TK_OUTPUTTYPE           183
-#define TK_STREAM               184
+#define TK_AGGREGATE            184
-#define TK_INTO                 185
+#define TK_BUFSIZE              185
-#define TK_TRIGGER              186
+#define TK_STREAM               186
-#define TK_AT_ONCE              187
+#define TK_INTO                 187
-#define TK_WINDOW_CLOSE         188
+#define TK_TRIGGER              188
-#define TK_IGNORE               189
+#define TK_AT_ONCE              189
-#define TK_EXPIRED              190
+#define TK_WINDOW_CLOSE         190
-#define TK_FILL_HISTORY         191
+#define TK_IGNORE               191
-#define TK_SUBTABLE             192
+#define TK_EXPIRED              192
-#define TK_KILL                 193
+#define TK_FILL_HISTORY         193
-#define TK_CONNECTION           194
+#define TK_SUBTABLE             194
-#define TK_TRANSACTION          195
+#define TK_KILL                 195
-#define TK_BALANCE              196
+#define TK_CONNECTION           196
-#define TK_VGROUP               197
+#define TK_TRANSACTION          197
-#define TK_MERGE                198
+#define TK_BALANCE              198
-#define TK_REDISTRIBUTE         199
+#define TK_VGROUP               199
-#define TK_SPLIT                200
+#define TK_MERGE                200
-#define TK_DELETE               201
+#define TK_REDISTRIBUTE         201
-#define TK_INSERT               202
+#define TK_SPLIT                202
-#define TK_NULL                 203
+#define TK_DELETE               203
-#define TK_NK_QUESTION          204
+#define TK_INSERT               204
-#define TK_NK_ARROW             205
+#define TK_NULL                 205
-#define TK_ROWTS                206
+#define TK_NK_QUESTION          206
-#define TK_QSTART               207
+#define TK_NK_ARROW             207
-#define TK_QEND                 208
+#define TK_ROWTS                208
-#define TK_QDURATION            209
+#define TK_QSTART               209
-#define TK_WSTART               210
+#define TK_QEND                 210
-#define TK_WEND                 211
+#define TK_QDURATION            211
-#define TK_WDURATION            212
+#define TK_WSTART               212
-#define TK_IROWTS               213
+#define TK_WEND                 213
-#define TK_CAST                 214
+#define TK_WDURATION            214
-#define TK_NOW                  215
+#define TK_IROWTS               215
-#define TK_TODAY                216
+#define TK_CAST                 216
-#define TK_TIMEZONE             217
+#define TK_NOW                  217
-#define TK_CLIENT_VERSION       218
+#define TK_TODAY                218
-#define TK_SERVER_VERSION       219
+#define TK_TIMEZONE             219
-#define TK_SERVER_STATUS        220
+#define TK_CLIENT_VERSION       220
-#define TK_CURRENT_USER         221
+#define TK_SERVER_VERSION       221
-#define TK_COUNT                222
+#define TK_SERVER_STATUS        222
-#define TK_LAST_ROW             223
+#define TK_CURRENT_USER         223
-#define TK_CASE                 224
+#define TK_COUNT                224
-#define TK_END                  225
+#define TK_LAST_ROW             225
-#define TK_WHEN                 226
+#define TK_CASE                 226
-#define TK_THEN                 227
+#define TK_END                  227
-#define TK_ELSE                 228
+#define TK_WHEN                 228
-#define TK_BETWEEN              229
+#define TK_THEN                 229
-#define TK_IS                   230
+#define TK_ELSE                 230
-#define TK_NK_LT                231
+#define TK_BETWEEN              231
-#define TK_NK_GT                232
+#define TK_IS                   232
-#define TK_NK_LE                233
+#define TK_NK_LT                233
-#define TK_NK_GE                234
+#define TK_NK_GT                234
-#define TK_NK_NE                235
+#define TK_NK_LE                235
-#define TK_MATCH                236
+#define TK_NK_GE                236
-#define TK_NMATCH               237
+#define TK_NK_NE                237
-#define TK_CONTAINS             238
+#define TK_MATCH                238
-#define TK_IN                   239
+#define TK_NMATCH               239
-#define TK_JOIN                 240
+#define TK_CONTAINS             240
-#define TK_INNER                241
+#define TK_IN                   241
-#define TK_SELECT               242
+#define TK_JOIN                 242
-#define TK_DISTINCT             243
+#define TK_INNER                243
-#define TK_WHERE                244
+#define TK_SELECT               244
-#define TK_PARTITION            245
+#define TK_DISTINCT             245
-#define TK_BY                   246
+#define TK_WHERE                246
-#define TK_SESSION              247
+#define TK_PARTITION            247
-#define TK_STATE_WINDOW         248
+#define TK_BY                   248
-#define TK_SLIDING              249
+#define TK_SESSION              249
-#define TK_FILL                 250
+#define TK_STATE_WINDOW         250
-#define TK_VALUE                251
+#define TK_SLIDING              251
-#define TK_NONE                 252
+#define TK_FILL                 252
-#define TK_PREV                 253
+#define TK_VALUE                253
-#define TK_LINEAR               254
+#define TK_NONE                 254
-#define TK_NEXT                 255
+#define TK_PREV                 255
-#define TK_HAVING               256
+#define TK_LINEAR               256
-#define TK_RANGE                257
+#define TK_NEXT                 257
-#define TK_EVERY                258
+#define TK_HAVING               258
-#define TK_ORDER                259
+#define TK_RANGE                259
-#define TK_SLIMIT               260
+#define TK_EVERY                260
-#define TK_SOFFSET              261
+#define TK_ORDER                261
-#define TK_LIMIT                262
+#define TK_SLIMIT               262
-#define TK_OFFSET               263
+#define TK_SOFFSET              263
-#define TK_ASC                  264
+#define TK_LIMIT                264
-#define TK_NULLS                265
+#define TK_OFFSET               265
-#define TK_ABORT                266
+#define TK_ASC                  266
-#define TK_AFTER                267
+#define TK_NULLS                267
-#define TK_ATTACH               268
+#define TK_ABORT                268
-#define TK_BEFORE               269
+#define TK_AFTER                269
-#define TK_BEGIN                270
+#define TK_ATTACH               270
-#define TK_BITAND               271
+#define TK_BEFORE               271
-#define TK_BITNOT               272
+#define TK_BEGIN                272
-#define TK_BITOR                273
+#define TK_BITAND               273
-#define TK_BLOCKS               274
+#define TK_BITNOT               274
-#define TK_CHANGE               275
+#define TK_BITOR                275
-#define TK_COMMA                276
+#define TK_BLOCKS               276
-#define TK_COMPACT              277
+#define TK_CHANGE               277
-#define TK_CONCAT               278
+#define TK_COMMA                278
-#define TK_CONFLICT             279
+#define TK_COMPACT              279
-#define TK_COPY                 280
+#define TK_CONCAT               280
-#define TK_DEFERRED             281
+#define TK_CONFLICT             281
-#define TK_DELIMITERS           282
+#define TK_COPY                 282
-#define TK_DETACH               283
+#define TK_DEFERRED             283
-#define TK_DIVIDE               284
+#define TK_DELIMITERS           284
-#define TK_DOT                  285
+#define TK_DETACH               285
-#define TK_EACH                 286
+#define TK_DIVIDE               286
-#define TK_FAIL                 287
+#define TK_DOT                  287
-#define TK_FILE                 288
+#define TK_EACH                 288
-#define TK_FOR                  289
+#define TK_FAIL                 289
-#define TK_GLOB                 290
+#define TK_FILE                 290
-#define TK_ID                   291
+#define TK_FOR                  291
-#define TK_IMMEDIATE            292
+#define TK_GLOB                 292
-#define TK_IMPORT               293
+#define TK_ID                   293
-#define TK_INITIALLY            294
+#define TK_IMMEDIATE            294
-#define TK_INSTEAD              295
+#define TK_IMPORT               295
-#define TK_ISNULL               296
+#define TK_INITIALLY            296
-#define TK_KEY                  297
+#define TK_INSTEAD              297
-#define TK_MODULES              298
+#define TK_ISNULL               298
-#define TK_NK_BITNOT            299
+#define TK_KEY                  299
-#define TK_NK_SEMI              300
+#define TK_MODULES              300
-#define TK_NOTNULL              301
+#define TK_NK_BITNOT            301
-#define TK_OF                   302
+#define TK_NK_SEMI              302
-#define TK_PLUS                 303
+#define TK_NOTNULL              303
-#define TK_PRIVILEGE            304
+#define TK_OF                   304
-#define TK_RAISE                305
+#define TK_PLUS                 305
-#define TK_REPLACE              306
+#define TK_PRIVILEGE            306
-#define TK_RESTRICT             307
+#define TK_RAISE                307
-#define TK_ROW                  308
+#define TK_REPLACE              308
-#define TK_SEMI                 309
+#define TK_RESTRICT             309
-#define TK_STAR                 310
+#define TK_ROW                  310
-#define TK_STATEMENT            311
+#define TK_SEMI                 311
-#define TK_STRING               312
+#define TK_STAR                 312
-#define TK_TIMES                313
+#define TK_STATEMENT            313
-#define TK_UPDATE               314
+#define TK_STRING               314
-#define TK_VALUES               315
+#define TK_TIMES                315
-#define TK_VARIABLE             316
+#define TK_UPDATE               316
-#define TK_VIEW                 317
+#define TK_VALUES               317
-#define TK_WAL                  318
+#define TK_VARIABLE             318
+#define TK_VIEW                 319
+#define TK_WAL                  320
 
 #define TK_NK_SPACE   600
 #define TK_NK_COMMENT 601

include/libs/executor/executor.h

@@ -160,13 +160,6 @@ int32_t qAsyncKillTask(qTaskInfo_t tinfo);
  */
 void qDestroyTask(qTaskInfo_t tinfo);
 
-/**
- * Get the queried table uid
- * @param qHandle
- * @return
- */
-int64_t qGetQueriedTableUid(qTaskInfo_t tinfo);
-
 /**
  * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks.
  *

include/libs/nodes/cmdnodes.h

@@ -45,6 +45,7 @@ extern "C" {
 #define PRIVILEGE_TYPE_ALL       PRIVILEGE_TYPE_MASK(0)
 #define PRIVILEGE_TYPE_READ      PRIVILEGE_TYPE_MASK(1)
 #define PRIVILEGE_TYPE_WRITE     PRIVILEGE_TYPE_MASK(2)
+#define PRIVILEGE_TYPE_SUBSCRIBE PRIVILEGE_TYPE_MASK(3)
 
 #define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 
@@ -423,7 +424,7 @@ typedef struct SDropFunctionStmt {
 typedef struct SGrantStmt {
   ENodeType type;
   char      userName[TSDB_USER_LEN];
-  char      dbName[TSDB_DB_NAME_LEN];
+  char      objName[TSDB_DB_NAME_LEN];  // db or topic
   int64_t   privileges;
 } SGrantStmt;
 

include/libs/nodes/nodes.h

@@ -187,6 +187,7 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_TRANSACTIONS_STMT,
   QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT,
   QUERY_NODE_SHOW_VNODES_STMT,
+  QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
   QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
   QUERY_NODE_SHOW_CREATE_TABLE_STMT,
   QUERY_NODE_SHOW_CREATE_STABLE_STMT,

include/libs/sync/sync.h

@@ -40,6 +40,8 @@ extern "C" {
 #define SNAPSHOT_MAX_CLOCK_SKEW_MS   1000 * 10
 #define SNAPSHOT_WAIT_MS             1000 * 30
 
+#define SYNC_MAX_RETRY_BACKOFF         5
+#define SYNC_LOG_REPL_RETRY_WAIT_MS    100
 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
 #define SYNC_HEART_TIMEOUT_MS          1000 * 8
 
@@ -49,7 +51,7 @@ extern "C" {
 #define SYNC_MAX_BATCH_SIZE 1
 #define SYNC_INDEX_BEGIN    0
 #define SYNC_INDEX_INVALID  -1
-#define SYNC_TERM_INVALID   0xFFFFFFFFFFFFFFFF
+#define SYNC_TERM_INVALID   -1  // 0xFFFFFFFFFFFFFFFF
 
 typedef enum {
   SYNC_STRATEGY_NO_SNAPSHOT = 0,
@@ -60,13 +62,14 @@ typedef enum {
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
 typedef int64_t  SyncIndex;
-typedef uint64_t SyncTerm;
+typedef int64_t  SyncTerm;
 
 typedef struct SSyncNode      SSyncNode;
 typedef struct SWal           SWal;
 typedef struct SSyncRaftEntry SSyncRaftEntry;
 
 typedef enum {
+  TAOS_SYNC_STATE_OFFLINE = 0,
   TAOS_SYNC_STATE_FOLLOWER = 100,
   TAOS_SYNC_STATE_CANDIDATE = 101,
   TAOS_SYNC_STATE_LEADER = 102,
@@ -135,13 +138,13 @@ typedef struct SSnapshotMeta {
 typedef struct SSyncFSM {
   void* data;
 
-  void (*FpCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
+  void (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
-  void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
+  void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
-  void (*FpRollBackCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
+  void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
 
   void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm);
-  void (*FpReConfigCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SReConfigCbMeta* pMeta);
+  void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta);
-  void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
+  void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
   bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm);
   int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm);
 
@@ -210,15 +213,20 @@ typedef struct SSyncInfo {
   int32_t (*syncEqCtrlMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg);
 } SSyncInfo;
 
+// if state == leader
+//     if restored, display "leader"
+//     if !restored && canRead, display "leader*"
+//     if !restored && !canRead, display "leader**"
 typedef struct SSyncState {
   ESyncState state;
   bool       restored;
+  bool       canRead;
 } SSyncState;
 
 int32_t syncInit();
 void    syncCleanUp();
 int64_t syncOpen(SSyncInfo* pSyncInfo);
-void    syncStart(int64_t rid);
+int32_t syncStart(int64_t rid);
 void    syncStop(int64_t rid);
 void    syncPreStop(int64_t rid);
 int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak);

include/libs/wal/wal.h

@@ -170,7 +170,7 @@ int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo
 
 // Assign version automatically and return to caller,
 // -1 will be returned for failed writes
-int64_t walAppendLog(SWal *, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
+int64_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
 
 void walFsync(SWal *, bool force);
 

include/os/osSocket.h

@@ -169,6 +169,9 @@ void        taosSetMaskSIGPIPE();
 uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len);
 
+uint64_t taosHton64(uint64_t val);
+uint64_t taosNtoh64(uint64_t val);
+
 #ifdef __cplusplus
 }
 #endif

include/os/osTime.h

@@ -35,6 +35,7 @@ extern "C" {
 #ifdef WINDOWS
 
 #define CLOCK_REALTIME 0
+#define CLOCK_MONOTONIC 0
 
 #define MILLISECOND_PER_SECOND (1000i64)
 #else
@@ -82,6 +83,13 @@ static FORCE_INLINE int64_t taosGetTimestampNs() {
   return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec;
 }
 
+//@return timestamp of monotonic clock in millisecond
+static FORCE_INLINE int64_t taosGetMonoTimestampMs() {
+  struct timespec systemTime = {0};
+  taosClockGetTime(CLOCK_MONOTONIC, &systemTime);
+  return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_nsec / 1000000;
+}
+
 char      *taosStrpTime(const char *buf, const char *fmt, struct tm *tm);
 struct tm *taosLocalTime(const time_t *timep, struct tm *result);
 struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst);

include/util/tRealloc.h

@@ -52,11 +52,13 @@ _exit:
   return code;
 }
 
-static FORCE_INLINE void tFree(uint8_t *pBuf) {
+#define tFree(BUF)                                        \
-  if (pBuf) {
+  do {                                                    \
-    taosMemoryFree(pBuf - sizeof(int64_t));
+    if (BUF) {                                            \
-  }
+      taosMemoryFree((uint8_t *)(BUF) - sizeof(int64_t)); \
-}
+      (BUF) = NULL;                                       \
+    }                                                     \
+  } while (0)
 
 #ifdef __cplusplus
 }

include/util/tarray.h

@@ -217,15 +217,10 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
-/**
- * clear the array (remove all element)
- * @param pArray
- * @param fp
- */
-void taosArrayClearP(SArray* pArray, FDelete fp);
-
 void* taosArrayDestroy(SArray* pArray);
+
 void  taosArrayDestroyP(SArray* pArray, FDelete fp);
+
 void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
 
 /**
@@ -235,12 +230,6 @@ void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
  */
 void taosArraySort(SArray* pArray, __compar_fn_t comparFn);
 
-/**
- * sort string array
- * @param pArray
- */
-void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
-
 /**
  * search the array
  * @param pArray

include/util/tdef.h

@@ -281,6 +281,7 @@ typedef enum ELogicConditionType {
 #define TSDB_DNODE_ROLE_VNODE 2
 
 #define TSDB_MAX_REPLICA 5
+#define TSDB_SYNC_LOG_BUFFER_SIZE 4096
 
 #define TSDB_TBNAME_COLUMN_INDEX     (-1)
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta

source/client/src/clientSml.c

@@ -79,7 +79,6 @@
 #define NCHAR_ADD_LEN  3  // L"nchar"   3 means L" "
 
 #define MAX_RETRY_TIMES 5
-#define LINE_BATCH      2000
 //=================================================================================================
 typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
 
@@ -467,6 +466,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
         goto end;
       }
       info->cost.numOfCreateSTables++;
+      taosMemoryFreeClear(pTableMeta);
+
+      code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
+      if (code != TSDB_CODE_SUCCESS) {
+        uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
+        goto end;
+      }
     } else if (code == TSDB_CODE_SUCCESS) {
       hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true,
                              HASH_NO_LOCK);
@@ -505,7 +511,6 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
           uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
           goto end;
         }
-      }
 
         taosMemoryFreeClear(pTableMeta);
         code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
@@ -516,6 +521,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
         if (code != TSDB_CODE_SUCCESS) {
           goto end;
         }
+      }
 
       taosHashClear(hashTmp);
       for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
@@ -552,12 +558,18 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
           uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
           goto end;
         }
-      }
 
         code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
         if (code != TSDB_CODE_SUCCESS) {
           goto end;
         }
+        code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
+        if (code != TSDB_CODE_SUCCESS) {
+          uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
+          goto end;
+        }
+      }
+
       needCheckMeta = true;
       taosHashCleanup(hashTmp);
       hashTmp = NULL;
@@ -565,13 +577,6 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
       uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code));
       goto end;
     }
-    taosMemoryFreeClear(pTableMeta);
-
-    code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
-    if (code != TSDB_CODE_SUCCESS) {
-      uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
-      goto end;
-    }
 
     if (needCheckMeta) {
       code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags,
@@ -596,7 +601,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
 end:
   taosHashCleanup(hashTmp);
   taosMemoryFreeClear(pTableMeta);
-  catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
+//  catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
   return code;
 }
 
@@ -815,6 +820,11 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) {
 }
 
 static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) {
+  void *tmp = taosMemoryCalloc(1, len + 1);
+  memcpy(tmp, data, len);
+  uDebug("SML:0x%" PRIx64 " smlParseInfluxTime tslen:%d, ts:%s", info->id, len, (char*)tmp);
+  taosMemoryFree(tmp);
+
   if (len == 0 || (len == 1 && data[0] == '0')) {
     return taosGetTimestampNs();
   }
@@ -2066,7 +2076,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *
 
 static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) {
   SSmlLineInfo elements = {0};
-  uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s", info->id, (info->isRawLine ? "rawdata" : sql));
+  void *tmp = taosMemoryCalloc(1, len + 1);
+  memcpy(tmp, sql, len);
+  uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? (char*)tmp : sql));
+  taosMemoryFree(tmp);
 
   int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf);
   if (ret != TSDB_CODE_SUCCESS) {
@@ -2562,7 +2575,7 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char
     goto end;
   }
 
-  batchs = ceil(((double)numLines) / LINE_BATCH);
+  batchs = ceil(((double)numLines) / tsSmlBatchSize);
   params.total = batchs;
   for (int i = 0; i < batchs; ++i) {
     SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT, 0);
@@ -2581,7 +2594,7 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char
     info->isRawLine = (rawLine == NULL);
     info->ttl       = ttl;
 
-    int32_t perBatch = LINE_BATCH;
+    int32_t perBatch = tsSmlBatchSize;
 
     if (numLines > perBatch) {
       numLines -= perBatch;

source/common/src/systable.c

@@ -206,16 +206,15 @@ static const SSysDbTableSchema vgroupsSchema[] = {
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
-    {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
+    {.name = "v1_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
-    {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "v1_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
+    {.name = "v2_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
-    {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "v2_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
+    {.name = "v3_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
-    {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "v3_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
+    {.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
-    {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
-    {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
 };
 
@@ -274,6 +273,12 @@ static const SSysDbTableSchema vnodesSchema[] = {
     {.name = "dnode_ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
+static const SSysDbTableSchema userUserPrivilegesSchema[] = {
+    {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "object_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+};
+
 static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
     {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@@ -298,6 +303,7 @@ static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false},
     {TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false},
     {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
+    {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), false},
 };
 
 static const SSysDbTableSchema connectionsSchema[] = {

source/common/src/tdataformat.c

@@ -841,11 +841,11 @@ _exit:
   return &pIter->cv;
 }
 
-static int32_t tRowAppendNoneToColData(SArray *aColData, int32_t nColData) {
+static int32_t tRowAppendNoneToColData(SColData *aColData, int32_t nColData) {
   int32_t code = 0;
 
   for (int32_t iColData = 0; iColData < nColData; iColData++) {
-    SColData *pColData = taosArrayGet(aColData, iColData);
+    SColData *pColData = &aColData[iColData];
     code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
     if (code) goto _exit;
   }
@@ -853,11 +853,11 @@ static int32_t tRowAppendNoneToColData(SArray *aColData, int32_t nColData) {
 _exit:
   return code;
 }
-static int32_t tRowAppendNullToColData(SArray *aColData, int32_t nColData, STSchema *pSchema) {
+static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STSchema *pSchema) {
   int32_t code = 0;
 
   int32_t   iColData = 0;
-  SColData *pColData = taosArrayGet(aColData, iColData);
+  SColData *pColData = &aColData[iColData];
   int32_t   iTColumn = 1;
   STColumn *pTColumn = &pSchema->columns[iTColumn];
 
@@ -866,12 +866,12 @@ static int32_t tRowAppendNullToColData(SArray *aColData, int32_t nColData, STSch
       if (pTColumn->colId == pColData->cid) {  // NULL
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
         if (code) goto _exit;
-        pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+        pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
         pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL;
       } else if (pTColumn->colId > pColData->cid) {  // NONE
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
         if (code) goto _exit;
-        pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+        pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
       } else {
         pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL;
       }
@@ -879,18 +879,18 @@ static int32_t tRowAppendNullToColData(SArray *aColData, int32_t nColData, STSch
       code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
       if (code) goto _exit;
 
-      pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+      pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
     }
   }
 
 _exit:
   return code;
 }
-static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SArray *aColData, int32_t nColData) {
+static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
   int32_t code = 0;
 
   int32_t   iColData = 0;
-  SColData *pColData = taosArrayGet(aColData, iColData);
+  SColData *pColData = &aColData[iColData];
   int32_t   iTColumn = 1;
   STColumn *pTColumn = &pTSchema->columns[iTColumn];
 
@@ -971,13 +971,13 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SArray *
         }
 
       _continue:
-        pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+        pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
         pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
       } else if (pTColumn->colId > pColData->cid) {  // NONE
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
         if (code) goto _exit;
 
-        pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+        pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
       } else {
         pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
       }
@@ -985,20 +985,20 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SArray *
       code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
       if (code) goto _exit;
 
-      pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+      pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
     }
   }
 
 _exit:
   return code;
 }
-static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SArray *aColData, int32_t nColData) {
+static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
   int32_t code = 0;
 
   SKVIdx   *pKVIdx = (SKVIdx *)pRow->data;
   uint8_t  *pv = NULL;
   int32_t   iColData = 0;
-  SColData *pColData = taosArrayGet(aColData, iColData);
+  SColData *pColData = &aColData[iColData];
   int32_t   iTColumn = 1;
   STColumn *pTColumn = &pTSchema->columns[iTColumn];
   int32_t   iCol = 0;
@@ -1054,26 +1054,26 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SArray *aCo
         if (code) goto _exit;
 
       _continue:
-        pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+        pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
         pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
       } else if (pTColumn->colId > pColData->cid) {
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
         if (code) goto _exit;
-        pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+        pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
       } else {
         pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
       }
     } else {
       code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
       if (code) goto _exit;
-      pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL;
+      pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
     }
   }
 
 _exit:
   return code;
 }
-int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SArray *aColData, int32_t nColData) {
+int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
   ASSERT(pRow->sver == pTSchema->version);
   ASSERT(nColData > 0);
 
@@ -1529,7 +1529,7 @@ void tColDataDestroy(void *ph) {
   SColData *pColData = (SColData *)ph;
 
   tFree(pColData->pBitMap);
-  tFree((uint8_t *)pColData->aOffset);
+  tFree(pColData->aOffset);
   tFree(pColData->pData);
 }
 
@@ -2018,41 +2018,60 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) {
   return v;
 }
 
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
+int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg) {
   int32_t code = 0;
-  int32_t size;
 
-  ASSERT(pColDataSrc->nVal > 0);
+  *pColData = *pColDataFrom;
-  ASSERT(pColDataDest->cid == pColDataSrc->cid);
-  ASSERT(pColDataDest->type == pColDataSrc->type);
-
-  pColDataDest->smaOn = pColDataSrc->smaOn;
-  pColDataDest->nVal = pColDataSrc->nVal;
-  pColDataDest->flag = pColDataSrc->flag;
 
   // bitmap
-  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
+  switch (pColData->flag) {
-    size = BIT2_SIZE(pColDataSrc->nVal);
+    case (HAS_NULL | HAS_NONE):
-    code = tRealloc(&pColDataDest->pBitMap, size);
+    case (HAS_VALUE | HAS_NONE):
-    if (code) goto _exit;
+    case (HAS_VALUE | HAS_NULL):
-    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
+      pColData->pBitMap = xMalloc(arg, BIT1_SIZE(pColData->nVal));
+      if (pColData->pBitMap == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _exit;
+      }
+      memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal));
+      break;
+    case (HAS_VALUE | HAS_NULL | HAS_NONE):
+      pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal));
+      if (pColData->pBitMap == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _exit;
+      }
+      memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal));
+      break;
+    default:
+      pColData->pBitMap = NULL;
+      break;
   }
 
   // offset
-  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
+  if (IS_VAR_DATA_TYPE(pColData->type) && (pColData->flag & HAS_VALUE)) {
-    size = sizeof(int32_t) * pColDataSrc->nVal;
+    pColData->aOffset = xMalloc(arg, pColData->nVal << 2);
-
+    if (pColData->aOffset == NULL) {
-    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
+      code = TSDB_CODE_OUT_OF_MEMORY;
-    if (code) goto _exit;
+      goto _exit;
-
+    }
-    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
+    memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2);
+  } else {
+    pColData->aOffset = NULL;
   }
 
   // value
-  pColDataDest->nData = pColDataSrc->nData;
+  if (pColData->nVal) {
-  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
+    pColData->pData = xMalloc(arg, pColData->nVal);
-  if (code) goto _exit;
+    if (pColData->pData == NULL) {
-  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _exit;
+    }
+
+    memcpy(pColData->pData, pColDataFrom->pData, pColData->nData);
+  } else {
+    pColData->pData = NULL;
+  }
 
 _exit:
   return code;

source/common/src/tglobal.c

@@ -75,6 +75,7 @@ char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = "";  // user defined child table
                                                      // If set to empty system will generate table name using MD5 hash.
 // true means that the name and order of cols in each line are the same(only for influx protocol)
 bool tsSmlDataFormat = false;
+int32_t  tsSmlBatchSize  = 10000;
 
 // query
 int32_t tsQueryPolicy = 1;
@@ -306,6 +307,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
   if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
   if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1;
   if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1;
@@ -648,6 +650,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
   tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
 
+  tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32;
   tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
 
   tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
@@ -1021,6 +1024,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
         tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
       } else if (strcasecmp("smlDataFormat", name) == 0) {
         tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
+      } else if (strcasecmp("smlBatchSize", name) == 0) {
+        tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32;
       } else if (strcasecmp("shellActivityTimer", name) == 0) {
         tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
       } else if (strcasecmp("supportVnodes", name) == 0) {

source/common/src/tmsg.c

@@ -992,15 +992,20 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
   if (tEncodeI32(&encoder, vlen) < 0) return -1;
   for (int32_t i = 0; i < vlen; ++i) {
     SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
+    int64_t reserved = 0;
     if (tEncodeI32(&encoder, pload->vgId) < 0) return -1;
     if (tEncodeI8(&encoder, pload->syncState) < 0) return -1;
     if (tEncodeI8(&encoder, pload->syncRestore) < 0) return -1;
+    if (tEncodeI8(&encoder, pload->syncCanRead) < 0) return -1;
     if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1;
     if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1;
     if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1;
     if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1;
     if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1;
     if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1;
+    if (tEncodeI64(&encoder, reserved) < 0) return -1;
+    if (tEncodeI64(&encoder, reserved) < 0) return -1;
+    if (tEncodeI64(&encoder, reserved) < 0) return -1;
   }
 
   // mnode loads
@@ -1065,15 +1070,20 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
 
   for (int32_t i = 0; i < vlen; ++i) {
     SVnodeLoad vload = {0};
+    int64_t    reserved = 0;
     if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1;
     if (tDecodeI8(&decoder, &vload.syncState) < 0) return -1;
     if (tDecodeI8(&decoder, &vload.syncRestore) < 0) return -1;
+    if (tDecodeI8(&decoder, &vload.syncCanRead) < 0) return -1;
     if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1;
     if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1;
     if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1;
     if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1;
     if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1;
     if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1;
+    if (tDecodeI64(&decoder, &reserved) < 0) return -1;
+    if (tDecodeI64(&decoder, &reserved) < 0) return -1;
+    if (tDecodeI64(&decoder, &reserved) < 0) return -1;
     if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -1288,7 +1298,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
   if (tEncodeI8(&encoder, pReq->enable) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->dbname) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1307,7 +1317,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
   if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->dbname) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -6737,7 +6747,7 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
       goto _exit;
     }
 
-    pSubmitTbData->aCol = taosArrayInit(nColData, nColData);
+    pSubmitTbData->aCol = taosArrayInit(nColData, sizeof(SColData));
     if (pSubmitTbData->aCol == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -150,7 +150,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
   SServerStatusRsp statusRsp = {0};
   SMonMloadInfo    minfo = {0};
   (*pMgmt->getMnodeLoadsFp)(&minfo);
-  if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) {
+  if (minfo.isMnode && (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) {
     pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED;
     snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState));
     return;
@@ -160,7 +160,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
   (*pMgmt->getVnodeLoadsFp)(&vinfo);
   for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) {
     SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i);
-    if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) {
+    if (pLoad->syncState == TAOS_SYNC_STATE_ERROR || pLoad->syncState == TAOS_SYNC_STATE_OFFLINE) {
       pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED;
       snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId,
                syncStr(pLoad->syncState));

source/dnode/mgmt/mgmt_mnode/src/mmWorker.c

@@ -162,11 +162,13 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
   if (pMsg == NULL) return -1;
   memcpy(pMsg, pRpc, sizeof(SRpcMsg));
+  pRpc->pCont = NULL;
 
   dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
   int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg);
   if (code != 0) {
     dTrace("msg:%p, is freed", pMsg);
+    rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
   }
   return code;

source/dnode/mgmt/mgmt_qnode/src/qmWorker.c

@@ -61,6 +61,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
   if (pMsg == NULL) return -1;
   memcpy(pMsg, pRpc, sizeof(SRpcMsg));
+  pRpc->pCont = NULL;
 
   switch (qtype) {
     case QUERY_QUEUE:
@@ -74,6 +75,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
       return 0;
     default:
       terrno = TSDB_CODE_INVALID_PARA;
+      rpcFreeCont(pMsg->pCont);
       taosFreeQitem(pMsg);
       return -1;
   }

source/dnode/mgmt/mgmt_snode/src/smWorker.c

@@ -151,6 +151,7 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   pHead->contLen = htonl(pHead->contLen);
   pHead->vgId = SNODE_HANDLE;
   memcpy(pMsg, pRpc, sizeof(SRpcMsg));
+  pRpc->pCont = NULL;
 
   switch (qtype) {
     case STREAM_QUEUE:

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -246,12 +246,12 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   pHead->contLen = htonl(pHead->contLen);
   pHead->vgId = htonl(pHead->vgId);
   memcpy(pMsg, pRpc, sizeof(SRpcMsg));
+  pRpc->pCont = NULL;
 
   int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype);
   if (code != 0) {
     dTrace("msg:%p, is freed", pMsg);
     rpcFreeCont(pMsg->pCont);
-    pRpc->pCont = NULL;
     taosFreeQitem(pMsg);
   }
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -328,6 +328,7 @@ typedef struct {
   int32_t    dnodeId;
   ESyncState syncState;
   bool       syncRestore;
+  bool       syncCanRead;
 } SVnodeGid;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -36,8 +36,6 @@ int64_t  mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup);
 SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId);
 int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup);
 int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups);
-int32_t mndAddVnodeToVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray);
-int32_t mndRemoveVnodeFromVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid);
 int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid);
 int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup);
 int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -538,7 +538,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
 
   int32_t code = -1;
   SArray *newSub = subscribe.topicNames;
-  taosArraySortString(newSub, taosArrayCompareString);
+  taosArraySort(newSub, taosArrayCompareString);
   taosArrayRemoveDuplicateP(newSub, taosArrayCompareString, taosMemoryFree);
 
   int32_t newTopicNum = taosArrayGetSize(newSub);
@@ -850,7 +850,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
 
     // add to current topic
     taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
-    taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString);
+    taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString);
+
     // set status
     if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
       if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||

source/dnode/mnode/impl/src/mndDnode.c

@@ -375,14 +375,18 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
       }
       bool roleChanged = false;
       for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
-        if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
+        SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
-          if (pVgroup->vnodeGid[vg].syncState != pVload->syncState ||
+        if (pGid->dnodeId == statusReq.dnodeId) {
-              pVgroup->vnodeGid[vg].syncRestore != pVload->syncRestore) {
+          if (pGid->syncState != pVload->syncState || pGid->syncRestore != pVload->syncRestore ||
-            mInfo("vgId:%d, state changed by status msg, old state:%s restored:%d new state:%s restored:%d",
+              pGid->syncCanRead != pVload->syncCanRead) {
-                  pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore,
+            mInfo(
-                  syncStr(pVload->syncState), pVload->syncRestore);
+                "vgId:%d, state changed by status msg, old state:%s restored:%d canRead:%d new state:%s restored:%d "
-            pVgroup->vnodeGid[vg].syncState = pVload->syncState;
+                "canRead:%d",
-            pVgroup->vnodeGid[vg].syncRestore = pVload->syncRestore;
+                pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead,
+                syncStr(pVload->syncState), pVload->syncRestore, pVload->syncCanRead);
+            pGid->syncState = pVload->syncState;
+            pGid->syncRestore = pVload->syncRestore;
+            pGid->syncCanRead = pVload->syncCanRead;
             roleChanged = true;
           }
           break;

source/dnode/mnode/impl/src/mndMain.c

@@ -150,12 +150,16 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs)
 
     bool roleChanged = false;
     for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
-      if (pVgroup->vnodeGid[vg].dnodeId == dnodeId) {
+      SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
-        if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_ERROR) {
+      if (pGid->dnodeId == dnodeId) {
-          mInfo("vgId:%d, state changed by offline check, old state:%s restored:%d new state:error restored:0",
+        if (pGid->syncState != TAOS_SYNC_STATE_OFFLINE) {
-                pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore);
+          mInfo(
-          pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_ERROR;
+              "vgId:%d, state changed by offline check, old state:%s restored:%d canRead:%d new state:error restored:0 "
-          pVgroup->vnodeGid[vg].syncRestore = 0;
+              "canRead:0",
+              pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead);
+          pGid->syncState = TAOS_SYNC_STATE_OFFLINE;
+          pGid->syncRestore = 0;
+          pGid->syncCanRead = 0;
           roleChanged = true;
         }
         break;
@@ -491,6 +495,16 @@ void mndPreClose(SMnode *pMnode) {
   if (pMnode != NULL) {
     syncLeaderTransfer(pMnode->syncMgmt.sync);
     syncPreStop(pMnode->syncMgmt.sync);
+#if 0
+    while (syncSnapshotRecving(pMnode->syncMgmt.sync)) {
+      mInfo("vgId:1, snapshot is recving");
+      taosMsleep(300);
+    }
+    while (syncSnapshotSending(pMnode->syncMgmt.sync)) {
+      mInfo("vgId:1, snapshot is sending");
+      taosMsleep(300);
+    }
+#endif
   }
 }
 
@@ -747,7 +761,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
         tstrncpy(desc.status, "ready", sizeof(desc.status));
         pClusterInfo->vgroups_alive++;
       }
-      if (pVgid->syncState != TAOS_SYNC_STATE_ERROR) {
+      if (pVgid->syncState != TAOS_SYNC_STATE_ERROR && pVgid->syncState != TAOS_SYNC_STATE_OFFLINE) {
         pClusterInfo->vnodes_alive++;
       }
       pClusterInfo->vnodes_total++;

source/dnode/mnode/impl/src/mndMnode.c

@@ -185,7 +185,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
     return -1;
   }
 
-  pObj->syncState = TAOS_SYNC_STATE_ERROR;
+  pObj->syncState = TAOS_SYNC_STATE_OFFLINE;
   mndReloadSyncConfig(pSdb->pMnode);
   return 0;
 }

source/dnode/mnode/impl/src/mndSync.c

@@ -72,15 +72,11 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
   return code;
 }
 
-void mndSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
+void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
   SMnode    *pMnode = pFsm->data;
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
   SSdbRaw   *pRaw = pMsg->pCont;
 
-  // delete msg handle
-  SRpcMsg rpcMsg = {0};
-  rpcMsg.info = pMsg->info;
-
   int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw);
   pMgmt->errCode = pMeta->code;
   mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64
@@ -120,6 +116,12 @@ void mndSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMet
   }
 }
 
+void mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
+  mndProcessWriteMsg(pFsm, pMsg, pMeta);
+  rpcFreeCont(pMsg->pCont);
+  pMsg->pCont = NULL;
+}
+
 int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) {
   mInfo("start to read snapshot from sdb in atomic way");
   SMnode *pMnode = pFsm->data;
@@ -361,7 +363,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
 
 void mndSyncStart(SMnode *pMnode) {
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
-  syncStart(pMgmt->sync);
+  if (syncStart(pMgmt->sync) < 0) {
+    mError("vgId:1, failed to start sync, id:%" PRId64, pMgmt->sync);
+    return;
+  }
   mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync);
 }
 

source/dnode/mnode/impl/src/mndUser.c

@@ -507,14 +507,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
   }
 
   if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
-    if (strcmp(alterReq.dbname, "1.*") != 0) {
+    if (strcmp(alterReq.objname, "1.*") != 0) {
-      int32_t len = strlen(alterReq.dbname) + 1;
+      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
       if (pDb == NULL) {
         mndReleaseDb(pMnode, pDb);
         goto _OVER;
       }
-      if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
+      if (taosHashPut(newUser.readDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
         mndReleaseDb(pMnode, pDb);
         goto _OVER;
       }
@@ -531,14 +531,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
   }
 
   if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
-    if (strcmp(alterReq.dbname, "1.*") != 0) {
+    if (strcmp(alterReq.objname, "1.*") != 0) {
-      int32_t len = strlen(alterReq.dbname) + 1;
+      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
       if (pDb == NULL) {
         mndReleaseDb(pMnode, pDb);
         goto _OVER;
       }
-      if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
+      if (taosHashPut(newUser.writeDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
         mndReleaseDb(pMnode, pDb);
         goto _OVER;
       }
@@ -555,28 +555,28 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
   }
 
   if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
-    if (strcmp(alterReq.dbname, "1.*") != 0) {
+    if (strcmp(alterReq.objname, "1.*") != 0) {
-      int32_t len = strlen(alterReq.dbname) + 1;
+      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
       if (pDb == NULL) {
         mndReleaseDb(pMnode, pDb);
         goto _OVER;
       }
-      taosHashRemove(newUser.readDbs, alterReq.dbname, len);
+      taosHashRemove(newUser.readDbs, alterReq.objname, len);
     } else {
       taosHashClear(newUser.readDbs);
     }
   }
 
   if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
-    if (strcmp(alterReq.dbname, "1.*") != 0) {
+    if (strcmp(alterReq.objname, "1.*") != 0) {
-      int32_t len = strlen(alterReq.dbname) + 1;
+      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
+      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
       if (pDb == NULL) {
         mndReleaseDb(pMnode, pDb);
         goto _OVER;
       }
-      taosHashRemove(newUser.writeDbs, alterReq.dbname, len);
+      taosHashRemove(newUser.writeDbs, alterReq.objname, len);
     } else {
       taosHashClear(newUser.writeDbs);
     }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -179,6 +179,23 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) {
   pOld->hashEnd = pNew->hashEnd;
   pOld->replica = pNew->replica;
   pOld->isTsma = pNew->isTsma;
+  for (int32_t i = 0; i < pNew->replica; ++i) {
+    SVnodeGid *pNewGid = &pNew->vnodeGid[i];
+    for (int32_t j = 0; j < pOld->replica; ++j) {
+      SVnodeGid *pOldGid = &pOld->vnodeGid[j];
+      if (pNewGid->dnodeId == pOldGid->dnodeId) {
+        pNewGid->syncState = pOldGid->syncState;
+        pNewGid->syncRestore = pOldGid->syncRestore;
+        pNewGid->syncCanRead = pOldGid->syncCanRead;
+      }
+    }
+  }
+  pNew->numOfTables = pOld->numOfTables;
+  pNew->numOfTimeSeries = pOld->numOfTimeSeries;
+  pNew->totalStorage = pOld->totalStorage;
+  pNew->compStorage = pOld->compStorage;
+  pNew->pointsWritten = pOld->pointsWritten;
+  pNew->compact = pOld->compact;
   memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
   return 0;
 }
@@ -659,11 +676,12 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false);
 
-    // default 3 replica
+    // default 3 replica, add 1 replica if move vnode
-    for (int32_t i = 0; i < 3; ++i) {
+    for (int32_t i = 0; i < 4; ++i) {
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       if (i < pVgroup->replica) {
-        colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false);
+        int16_t dnodeId = (int16_t)pVgroup->vnodeGid[i].dnodeId;
+        colDataAppend(pColInfo, numOfRows, (const char *)&dnodeId, false);
 
         bool       exist = false;
         bool       online = false;
@@ -679,8 +697,16 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
         if (!exist) {
           strcpy(role, "dropping");
         } else if (online) {
-          bool show = (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER && !pVgroup->vnodeGid[i].syncRestore);
+          char *star = "";
-          snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), show ? "*" : "");
+          if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
+            if (!pVgroup->vnodeGid[i].syncRestore && !pVgroup->vnodeGid[i].syncCanRead) {
+              star = "**";
+            } else if (!pVgroup->vnodeGid[i].syncRestore && pVgroup->vnodeGid[i].syncCanRead) {
+              star = "*";
+            } else {
+            }
+          }
+          snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), star);
         } else {
         }
         STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);
@@ -695,16 +721,8 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
     }
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppendNULL(pColInfo, numOfRows);
+    int32_t cacheUsage = (int32_t)pVgroup->cacheUsage;
-
+    colDataAppend(pColInfo, numOfRows, (const char *)&cacheUsage, false);
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->cacheUsage, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppendNULL(pColInfo, numOfRows);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppendNULL(pColInfo, numOfRows);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
@@ -851,7 +869,7 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) {
   sdbCancelFetch(pSdb, pIter);
 }
 
-int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
+static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray) {
   taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
     SDnodeObj *pDnode = taosArrayGet(pArray, i);
@@ -887,12 +905,21 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
     }
 
     pVgid->dnodeId = pDnode->id;
-    pVgid->syncState = TAOS_SYNC_STATE_ERROR;
+    pVgid->syncState = TAOS_SYNC_STATE_OFFLINE;
     mInfo("db:%s, vgId:%d, vn:%d is added, memory:%" PRId64 ", dnode:%d avail:%" PRId64 " used:%" PRId64,
           pVgroup->dbName, pVgroup->vgId, pVgroup->replica, vgMem, pVgid->dnodeId, pDnode->memAvail, pDnode->memUsed);
 
     pVgroup->replica++;
     pDnode->numOfVnodes++;
+
+    SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
+    if (pVgRaw == NULL) return -1;
+    if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
+      sdbFreeRaw(pVgRaw);
+      return -1;
+    }
+    (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
+
     return 0;
   }
 
@@ -901,7 +928,8 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
   return -1;
 }
 
-int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) {
+static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray,
+                                        SVnodeGid *pDelVgid) {
   taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
     SDnodeObj *pDnode = taosArrayGet(pArray, i);
@@ -941,6 +969,15 @@ _OVER:
     SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
     mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
   }
+
+  SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
+  if (pVgRaw == NULL) return -1;
+  if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
+    sdbFreeRaw(pVgRaw);
+    return -1;
+  }
+  (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
+
   return 0;
 }
 
@@ -1088,7 +1125,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
 
   if (!force) {
     mInfo("vgId:%d, will add 1 vnode", pVgroup->vgId);
-    if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
+    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
     for (int32_t i = 0; i < newVg.replica - 1; ++i) {
       if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
     }
@@ -1100,6 +1137,16 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
     SVnodeGid del = newVg.vnodeGid[vnIndex];
     newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
     memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
+    {
+      SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
+      if (pRaw == NULL) return -1;
+      if (mndTransAppendRedolog(pTrans, pRaw) != 0) {
+        sdbFreeRaw(pRaw);
+        return -1;
+      }
+      (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    }
+
     if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
     for (int32_t i = 0; i < newVg.replica; ++i) {
       if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
@@ -1107,11 +1154,20 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
     if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
   } else {
     mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId);
-    if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
+    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
     newVg.replica--;
     SVnodeGid del = newVg.vnodeGid[vnIndex];
     newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
     memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
+    {
+      SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
+      if (pRaw == NULL) return -1;
+      if (mndTransAppendRedolog(pTrans, pRaw) != 0) {
+        sdbFreeRaw(pRaw);
+        return -1;
+      }
+      (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    }
 
     for (int32_t i = 0; i < newVg.replica; ++i) {
       if (i != vnIndex) {
@@ -1128,16 +1184,12 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
 
   {
     SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
-    if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1;
+    if (pRaw == NULL) return -1;
-    (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
-    pRaw = NULL;
+      sdbFreeRaw(pRaw);
+      return -1;
     }
-
-  {
-    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
-    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1;
     (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-    pRaw = NULL;
   }
 
   mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica);
@@ -1193,7 +1245,15 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb
   SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica];
   pVgroup->replica++;
   pGid->dnodeId = newDnodeId;
-  pGid->syncState = TAOS_SYNC_STATE_ERROR;
+  pGid->syncState = TAOS_SYNC_STATE_OFFLINE;
+
+  SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
+  if (pVgRaw == NULL) return -1;
+  if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
+    sdbFreeRaw(pVgRaw);
+    return -1;
+  }
+  (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
 
   for (int32_t i = 0; i < pVgroup->replica - 1; ++i) {
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
@@ -1224,6 +1284,14 @@ static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, S
   memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid));
   memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));
 
+  SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
+  if (pVgRaw == NULL) return -1;
+  if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
+    sdbFreeRaw(pVgRaw);
+    return -1;
+  }
+  (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
+
   if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1;
   for (int32_t i = 0; i < pVgroup->replica; ++i) {
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
@@ -1237,7 +1305,6 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
                                      SDnodeObj *pOld1, SDnodeObj *pNew2, SDnodeObj *pOld2, SDnodeObj *pNew3,
                                      SDnodeObj *pOld3) {
   int32_t code = -1;
-  SSdbRaw *pRaw = NULL;
   STrans *pTrans = NULL;
 
   pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup");
@@ -1319,17 +1386,13 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
   }
 
   {
-    pRaw = mndVgroupActionEncode(&newVg);
+    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
-    if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER;
+    if (pRaw == NULL) return -1;
-    (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
-    pRaw = NULL;
+      sdbFreeRaw(pRaw);
+      return -1;
     }
-
-  {
-    pRaw = mndVgroupActionEncode(&newVg);
-    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
     (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-    pRaw = NULL;
   }
 
   mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica);
@@ -1342,7 +1405,6 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
 
 _OVER:
   mndTransDrop(pTrans);
-  sdbFreeRaw(pRaw);
   mndReleaseDb(pMnode, pDb);
   return code;
 }
@@ -1593,13 +1655,13 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
     mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
           pVgroup->vnodeGid[0].dnodeId);
 
-    if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
+    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
       return -1;
     if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1;
     if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
 
-    if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
+    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
       return -1;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0)
@@ -1612,7 +1674,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
 
     SVnodeGid del1 = {0};
     SVnodeGid del2 = {0};
-    if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
+    if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del1) != 0) return -1;
     if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
       return -1;
@@ -1620,7 +1682,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
       return -1;
     if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
 
-    if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
+    if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del2) != 0) return -1;
     if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
       return -1;
@@ -1629,16 +1691,6 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
     return -1;
   }
 
-  {
-    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
-    if (pVgRaw == NULL) return -1;
-    if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
-      sdbFreeRaw(pVgRaw);
-      return -1;
-    }
-    (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
-  }
-
   {
     SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
     if (pVgRaw == NULL) return -1;
@@ -1658,7 +1710,6 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans,
 
 static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) {
   int32_t code = -1;
-  SSdbRaw *pRaw = NULL;
   STrans *pTrans = NULL;
   SArray *pArray = mndBuildDnodesArray(pMnode, 0);
 
@@ -1676,13 +1727,13 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
   }
 
   if (newVg1.replica == 1) {
-    if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER;
+    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
     if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER;
     if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
   } else if (newVg1.replica == 3) {
     SVnodeGid del1 = {0};
-    if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER;
+    if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER;
     if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
     if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER;
@@ -1727,17 +1778,23 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
 #endif
 
   {
-    pRaw = mndVgroupActionEncode(&newVg1);
+    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg1);
-    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
+    if (pRaw == NULL) return -1;
+    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
+      sdbFreeRaw(pRaw);
+      return -1;
+    }
     (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-    pRaw = NULL;
   }
 
   {
-    pRaw = mndVgroupActionEncode(&newVg2);
+    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg2);
-    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
+    if (pRaw == NULL) return -1;
+    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
+      sdbFreeRaw(pRaw);
+      return -1;
+    }
     (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-    pRaw = NULL;
   }
 
   mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
@@ -1757,7 +1814,6 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
 _OVER:
   taosArrayDestroy(pArray);
   mndTransDrop(pTrans);
-  sdbFreeRaw(pRaw);
   return code;
 }
 
@@ -1802,16 +1858,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
 
   {
     SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
-    if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) {
+    if (pRaw == NULL) return -1;
-      sdbFreeRaw(pRaw);
+    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
-      return -1;
-    }
-    (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-  }
-
-  {
-    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
-    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) {
       sdbFreeRaw(pRaw);
       return -1;
     }

source/dnode/vnode/src/inc/tsdb.h

@@ -70,6 +70,9 @@ typedef struct SDiskData        SDiskData;
 typedef struct SDiskDataBuilder SDiskDataBuilder;
 typedef struct SBlkInfo         SBlkInfo;
 
+#define TSDBROW_ROW_FMT ((int8_t)0x0)
+#define TSDBROW_COL_FMT ((int8_t)0x1)
+
 #define TSDB_FILE_DLMT     ((uint32_t)0xF00AFA0F)
 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
@@ -102,12 +105,14 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
 
 // tsdbUtil.c ==============================================================================================
 // TSDBROW
-#define TSDBROW_TS(ROW)                       (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
+#define TSDBROW_TS(ROW) (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
-#define TSDBROW_VERSION(ROW)                  (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow])
+#define TSDBROW_VERSION(ROW) \
+  (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow])
 #define TSDBROW_SVERSION(ROW)            TD_ROW_SVER((ROW)->pTSRow)
 #define TSDBROW_KEY(ROW)                 ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)})
-#define tsdbRowFromTSRow(VERSION, TSROW)      ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)})
+#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = TSDBROW_ROW_FMT, .version = (VERSION), .pTSRow = (TSROW)})
-#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
+#define tsdbRowFromBlockData(BLOCKDATA, IROW) \
+  ((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
 void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
 int32_t tsdbRowCmprFn(const void *p1, const void *p2);
@@ -150,17 +155,15 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs);
 #define tBlockDataLastRow(PBLOCKDATA)              tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
 #define tBlockDataFirstKey(PBLOCKDATA)             TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA))
 #define tBlockDataLastKey(PBLOCKDATA)              TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
+#define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX])
 
 int32_t tBlockDataCreate(SBlockData *pBlockData);
-void      tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear);
+void    tBlockDataDestroy(SBlockData *pBlockData);
 int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid);
 void    tBlockDataReset(SBlockData *pBlockData);
 int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
 void    tBlockDataClear(SBlockData *pBlockData);
-SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
 void    tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
-int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
-int32_t   tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData);
 int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
                        int32_t aBufN[]);
 int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
@@ -335,10 +338,13 @@ struct SVersionRange {
 typedef struct SMemSkipListNode SMemSkipListNode;
 struct SMemSkipListNode {
   int8_t            level;
+  int8_t            flag;  // TSDBROW_ROW_FMT for row format, TSDBROW_COL_FMT for col format
+  int32_t           iRow;
   int64_t           version;
-  SRow             *pTSRow;
+  void             *pData;
   SMemSkipListNode *forwards[0];
 };
+
 typedef struct SMemSkipList {
   int64_t           size;
   uint32_t          seed;
@@ -376,7 +382,7 @@ struct SMemTable {
 };
 
 struct TSDBROW {
-  int8_t type;  // 0 for row from tsRow, 1 for row from block data
+  int8_t type;  // TSDBROW_ROW_FMT for row from tsRow, TSDBROW_COL_FMT for row from block data
   union {
     struct {
       int64_t version;
@@ -473,7 +479,7 @@ struct SBlockData {
   int64_t  *aVersion;  // versions of each row
   TSKEY    *aTSKEY;    // timestamp of each row
   int32_t   nColData;
-  SArray  *aColData;  // SArray<SColData>
+  SColData *aColData;
 };
 
 struct TABLEID {
@@ -739,8 +745,8 @@ typedef struct {
 
 int32_t tsdbOpenCache(STsdb *pTsdb);
 void    tsdbCloseCache(STsdb *pTsdb);
-int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, SRow *row, STsdb *pTsdb);
+int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb);
-int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, SRow *row, bool dup);
+int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup);
 int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h);
 int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h);
 int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
@@ -795,8 +801,13 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
   }
 
   pIter->pRow = &pIter->row;
-  pIter->pRow->version = pIter->pNode->version;
+  if (pIter->pNode->flag == TSDBROW_ROW_FMT) {
-  pIter->pRow->pTSRow = pIter->pNode->pTSRow;
+    pIter->row = tsdbRowFromTSRow(pIter->pNode->version, pIter->pNode->pData);
+  } else if (pIter->pNode->flag == TSDBROW_COL_FMT) {
+    pIter->row = tsdbRowFromBlockData(pIter->pNode->pData, pIter->pNode->iRow);
+  } else {
+    ASSERT(0);
+  }
 
   return pIter->pRow;
 }

source/dnode/vnode/src/inc/vnd.h

@@ -97,7 +97,7 @@ bool    vnodeShouldRollback(SVnode* pVnode);
 
 // vnodeSync.c
 int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
-void    vnodeSyncStart(SVnode* pVnode);
+int32_t vnodeSyncStart(SVnode* pVnode);
 void    vnodeSyncPreClose(SVnode* pVnode);
 void    vnodeSyncClose(SVnode* pVnode);
 void    vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -87,7 +87,7 @@ typedef struct SSnapDataHdr       SSnapDataHdr;
 #define VNODE_RSMA2_DIR "rsma2"
 
 // vnd.h
-void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
+void* vnodeBufPoolMalloc(SVBufPool* pPool, int32_t size);
 void  vnodeBufPoolFree(SVBufPool* pPool, void* p);
 void  vnodeBufPoolRef(SVBufPool* pPool);
 void  vnodeBufPoolUnRef(SVBufPool* pPool);

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -190,7 +190,7 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) {
   return code;
 }
 
-int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, SRow *row, bool dup) {
+int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) {
   int32_t code = 0;
   STSRow *cacheRow = NULL;
   char    key[32] = {0};
@@ -201,7 +201,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, SR
   LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen);
   if (h) {
     STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
-    TSKEY     keyTs = row->ts;
+    TSKEY     keyTs = TSDBROW_TS(row);
     bool      invalidate = false;
 
     SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h);
@@ -222,7 +222,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, SR
         SColVal *tColVal = &tTsVal1->colVal;
 
         SColVal colVal = {0};
-        tRowGet(row, pTSchema, iCol, &colVal);
+        tsdbRowGetColVal(row, pTSchema, iCol, &colVal);
         if (!COL_VAL_IS_NONE(&colVal)) {
           if (keyTs == tTsVal1->ts && !COL_VAL_IS_NONE(tColVal)) {
             invalidate = true;
@@ -316,7 +316,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, SR
   return code;
 }
 
-int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, SRow *row, STsdb *pTsdb) {
+int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb) {
   int32_t code = 0;
   STSRow *cacheRow = NULL;
   char    key[32] = {0};
@@ -327,7 +327,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, SRow *row, STsdb *p
   LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen);
   if (h) {
     STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
-    TSKEY     keyTs = row->ts;
+    TSKEY     keyTs = TSDBROW_TS(row);
     bool      invalidate = false;
 
     SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h);
@@ -348,7 +348,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, SRow *row, STsdb *p
         SColVal *tColVal = &tTsVal1->colVal;
 
         SColVal colVal = {0};
-        tRowGet(row, pTSchema, iCol, &colVal);
+        tsdbRowGetColVal(row, pTSchema, iCol, &colVal);
         if (!COL_VAL_IS_VALUE(&colVal)) {
           if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) {
             invalidate = true;
@@ -672,7 +672,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       } else {
         // tBlockDataDestroy(&state->blockData, 1);
         if (state->pBlockData) {
-          tBlockDataDestroy(state->pBlockData, 1);
+          tBlockDataDestroy(state->pBlockData);
           state->pBlockData = NULL;
         }
 
@@ -794,7 +794,7 @@ _err:
     state->aBlockIdx = NULL;
   }
   if (state->pBlockData) {
-    tBlockDataDestroy(state->pBlockData, 1);
+    tBlockDataDestroy(state->pBlockData);
     state->pBlockData = NULL;
   }
 
@@ -821,7 +821,7 @@ int32_t clearNextRowFromFS(void *iter) {
   }
   if (state->pBlockData) {
     // tBlockDataDestroy(&state->blockData, 1);
-    tBlockDataDestroy(state->pBlockData, 1);
+    tBlockDataDestroy(state->pBlockData);
     state->pBlockData = NULL;
   }
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -811,7 +811,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
   int32_t lino = 0;
 
   memset(pCommitter, 0, sizeof(*pCommitter));
-  ASSERT(pTsdb->mem && pTsdb->imem == NULL);
+  ASSERT(pTsdb->mem && pTsdb->imem == NULL && "last tsdb commit incomplete");
 
   taosThreadRwlockWrlock(&pTsdb->rwLock);
   pTsdb->imem = pTsdb->mem;
@@ -903,24 +903,24 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
   // reader
   taosArrayDestroy(pCommitter->dReader.aBlockIdx);
   tMapDataClear(&pCommitter->dReader.mBlock);
-  tBlockDataDestroy(&pCommitter->dReader.bData, 1);
+  tBlockDataDestroy(&pCommitter->dReader.bData);
 
   // merger
   for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
     SDataIter *pIter = &pCommitter->aDataIter[iStt];
     taosArrayDestroy(pIter->aSttBlk);
-    tBlockDataDestroy(&pIter->bData, 1);
+    tBlockDataDestroy(&pIter->bData);
   }
 
   // writer
   taosArrayDestroy(pCommitter->dWriter.aBlockIdx);
   taosArrayDestroy(pCommitter->dWriter.aSttBlk);
   tMapDataClear(&pCommitter->dWriter.mBlock);
-  tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
+  tBlockDataDestroy(&pCommitter->dWriter.bData);
 #if USE_STREAM_COMPRESSION
   tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder);
 #else
-  tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
+  tBlockDataDestroy(&pCommitter->dWriter.bDatal);
 #endif
   tDestroyTSchema(pCommitter->skmTable.pTSchema);
   tDestroyTSchema(pCommitter->skmRow.pTSchema);
@@ -1173,7 +1173,6 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
 
   tBlockDataClear(pBlockData);
   while (pRowInfo) {
-    ASSERT(pRowInfo->row.type == 0);
     code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
     TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -1241,7 +1240,6 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
         pRow = NULL;
       }
     } else if (c > 0) {
-      ASSERT(pRowInfo->row.type == 0);
       code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
       TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -1261,7 +1259,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
         }
       }
     } else {
-      ASSERT(0);
+      ASSERT(0 && "dup rows not allowed");
     }
 
     if (pBDataW->nRow >= pCommitter->maxRow) {
@@ -1483,7 +1481,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
 
     while (pRowInfo) {
       STSchema *pTSchema = NULL;
-      if (pRowInfo->row.type == 0) {
+      if (pRowInfo->row.type == TSDBROW_ROW_FMT) {
         code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
         TSDB_CHECK_CODE(code, lino, _exit);
         pTSchema = pCommitter->skmRow.pTSchema;
@@ -1526,7 +1524,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
 
     while (pRowInfo) {
       STSchema *pTSchema = NULL;
-      if (pRowInfo->row.type == 0) {
+      if (pRowInfo->row.type == TSDBROW_ROW_FMT) {
         code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
         TSDB_CHECK_CODE(code, lino, _exit);
         pTSchema = pCommitter->skmRow.pTSchema;

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -28,7 +28,9 @@
 
 static void    tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
 static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
-static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version,
+static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version,
+                                        SSubmitTbData *pSubmitTbData, int32_t *affectedRows);
+static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version,
                                         SSubmitTbData *pSubmitTbData, int32_t *affectedRows);
 
 int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
@@ -133,10 +135,12 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitTbData *pSubmi
   }
 
   // do insert impl
-  code = tsdbInsertTableDataImpl(pMemTable, pTbData, version, pSubmitTbData, affectedRows);
+  if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
-  if (code) {
+    code = tsdbInsertColDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows);
-    goto _err;
+  } else {
+    code = tsdbInsertRowDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows);
   }
+  if (code) goto _err;
 
   return code;
 
@@ -243,7 +247,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDa
   pIter->pTbData = pTbData;
   pIter->backward = backward;
   pIter->pRow = NULL;
-  pIter->row.type = 0;
   if (pFrom == NULL) {
     // create from head or tail
     if (backward) {
@@ -411,8 +414,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p
       for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) {
         pn = SL_NODE_BACKWARD(px, iLevel);
         while (pn != pTbData->sl.pHead) {
+          if (pn->flag == TSDBROW_ROW_FMT) {
             tKey.version = pn->version;
-          tKey.ts = pn->pTSRow->ts;
+            tKey.ts = ((SRow *)pn->pData)->ts;
+          } else if (pn->flag == TSDBROW_COL_FMT) {
+            tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow];
+            tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow];
+          }
 
           int32_t c = tsdbKeyCmprFn(&tKey, pKey);
           if (c <= 0) {
@@ -441,8 +449,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p
       for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) {
         pn = SL_NODE_FORWARD(px, iLevel);
         while (pn != pTbData->sl.pTail) {
+          if (pn->flag == TSDBROW_ROW_FMT) {
             tKey.version = pn->version;
-          tKey.ts = pn->pTSRow->ts;
+            tKey.ts = ((SRow *)pn->pData)->ts;
+          } else if (pn->flag == TSDBROW_COL_FMT) {
+            tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow];
+            tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow];
+          }
 
           int32_t c = tsdbKeyCmprFn(&tKey, pKey);
           if (c >= 0) {
@@ -469,29 +482,39 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
 
   return level;
 }
-static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, int64_t version, SRow *pRow,
+static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, TSDBROW *pRow,
                            int8_t forward) {
   int32_t           code = 0;
   int8_t            level;
   SMemSkipListNode *pNode;
   SVBufPool        *pPool = pMemTable->pTsdb->pVnode->inUse;
 
+  ASSERT(pPool != NULL);
+
   // node
   level = tsdbMemSkipListRandLevel(&pTbData->sl);
-  ASSERT(pPool != NULL);
   pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level));
   if (pNode == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
   pNode->level = level;
-  pNode->version = version;
+  pNode->flag = pRow->type;
-  pNode->pTSRow = vnodeBufPoolMalloc(pPool, pRow->len);
+
-  if (NULL == pNode->pTSRow) {
+  if (pRow->type == TSDBROW_ROW_FMT) {
+    pNode->version = pRow->version;
+    pNode->pData = vnodeBufPoolMalloc(pPool, pRow->pTSRow->len);
+    if (NULL == pNode->pData) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
     }
-  memcpy(pNode->pTSRow, pRow, pRow->len);
+    memcpy(pNode->pData, pRow->pTSRow, pRow->pTSRow->len);
+  } else if (pRow->type == TSDBROW_COL_FMT) {
+    pNode->iRow = pRow->iRow;
+    pNode->pData = pRow->pBlockData;
+  } else {
+    ASSERT(0);
+  }
 
   for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) {
     SMemSkipListNode *pn = pos[iLevel];
@@ -538,31 +561,132 @@ _exit:
   return code;
 }
 
-static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version,
+static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version,
                                         SSubmitTbData *pSubmitTbData, int32_t *affectedRows) {
   int32_t code = 0;
 
-  SRow            **rows = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP);
+  SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
-  TSDBKEY           key = {.version = version};
+  int32_t    nColData = TARRAY_SIZE(pSubmitTbData->aCol);
-  SMemSkipListNode *pos[SL_MAX_LEVEL];
+  SColData  *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol);
-  TSDBROW           row = tsdbRowFromTSRow(version, NULL);
-  int32_t           nRow = TARRAY_SIZE(pSubmitTbData->aRowP);
-  int32_t           iRow = 0;
-  SRow             *pLastRow = NULL;
 
-  // backward put first data
+  ASSERT(aColData[0].cid = PRIMARYKEY_TIMESTAMP_COL_ID);
-  row.pTSRow = rows[iRow];
+  ASSERT(aColData[0].type = TSDB_DATA_TYPE_TIMESTAMP);
-  key.ts = row.pTSRow->ts;
+  ASSERT(aColData[0].flag = HAS_VALUE);
-  iRow++;
+
-  tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
+  // copy and construct block data
-  code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 0);
+  SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData));
-  if (code) {
+  if (pBlockData == NULL) {
-    goto _err;
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
   }
 
-  pTbData->minKey = TMIN(pTbData->minKey, key.ts);
+  pBlockData->suid = pTbData->suid;
+  pBlockData->uid = pTbData->uid;
+  pBlockData->nRow = aColData[0].nVal;
+  pBlockData->aUid = NULL;
+  pBlockData->aVersion = vnodeBufPoolMalloc(pPool, aColData[0].nData);
+  if (pBlockData->aVersion == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+  for (int32_t i = 0; i < pBlockData->nRow; i++) {  // todo: here can be optimized
+    pBlockData->aVersion[i] = version;
+  }
 
-  pLastRow = row.pTSRow;
+  pBlockData->aTSKEY = vnodeBufPoolMalloc(pPool, aColData[0].nData);
+  if (pBlockData->aTSKEY == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+  memcpy(pBlockData->aTSKEY, aColData[0].pData, aColData[0].nData);
+
+  pBlockData->nColData = nColData - 1;
+  pBlockData->aColData = vnodeBufPoolMalloc(pPool, sizeof(SColData) * pBlockData->nColData);
+  if (pBlockData->aColData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+  }
+  for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) {
+    code = tColDataCopy(&aColData[iColData + 1], &pBlockData->aColData[iColData], (xMallocFn)vnodeBufPoolMalloc, pPool);
+    if (code) goto _exit;
+  }
+
+  // loop to add each row to the skiplist
+  SMemSkipListNode *pos[SL_MAX_LEVEL];
+  TSDBROW           tRow = tsdbRowFromBlockData(pBlockData, 0);
+  TSDBKEY           key = {.version = version, .ts = pBlockData->aTSKEY[0]};
+  TSDBROW           lRow;  // last row
+
+  // first row
+  tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
+  if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit;
+  pTbData->minKey = TMIN(pTbData->minKey, key.ts);
+  lRow = tRow;
+
+  // remain row
+  ++tRow.iRow;
+  if (tRow.iRow < pBlockData->nRow) {
+    for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) {
+      pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel);
+    }
+
+    while (tRow.iRow < pBlockData->nRow) {
+      key.ts = pBlockData->aTSKEY[tRow.iRow];
+
+      if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) {
+        tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS);
+      }
+
+      if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit;
+      lRow = tRow;
+
+      ++tRow.iRow;
+    }
+  }
+
+  if (key.ts >= pTbData->maxKey) {
+    pTbData->maxKey = key.ts;
+
+    if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) {
+      tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true);
+    }
+  }
+
+  if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) {
+    tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb);
+  }
+
+  // SMemTable
+  pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey);
+  pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey);
+  pMemTable->nRow += pBlockData->nRow;
+
+  if (affectedRows) *affectedRows = pBlockData->nRow;
+
+_exit:
+  return code;
+}
+
+static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version,
+                                        SSubmitTbData *pSubmitTbData, int32_t *affectedRows) {
+  int32_t code = 0;
+
+  int32_t           nRow = TARRAY_SIZE(pSubmitTbData->aRowP);
+  SRow            **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP);
+  TSDBKEY           key = {.version = version};
+  SMemSkipListNode *pos[SL_MAX_LEVEL];
+  TSDBROW           tRow = {.type = TSDBROW_ROW_FMT, .version = version};
+  int32_t           iRow = 0;
+  TSDBROW           lRow;
+
+  // backward put first data
+  tRow.pTSRow = aRow[iRow++];
+  key.ts = tRow.pTSRow->ts;
+  tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
+  code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0);
+  if (code) goto _exit;
+  lRow = tRow;
+
+  pTbData->minKey = TMIN(pTbData->minKey, key.ts);
 
   // forward put rest data
   if (iRow < nRow) {
@@ -571,36 +695,32 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
     }
 
     while (iRow < nRow) {
-      row.pTSRow = rows[iRow];
+      tRow.pTSRow = aRow[iRow++];
-      key.ts = row.pTSRow->ts;
+      key.ts = tRow.pTSRow->ts;
 
       if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) {
         tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS);
       }
 
-      code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 1);
+      code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1);
-      if (code) {
+      if (code) goto _exit;
-        goto _err;
-      }
 
-      pLastRow = row.pTSRow;
+      lRow = tRow;
 
       iRow++;
     }
   }
 
   if (key.ts >= pTbData->maxKey) {
-    if (key.ts > pTbData->maxKey) {
     pTbData->maxKey = key.ts;
-    }
 
-    if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && pLastRow != NULL) {
+    if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) {
-      tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, pLastRow, true);
+      tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true);
     }
   }
 
   if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) {
-    tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb);
+    tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb);
   }
 
   // SMemTable
@@ -610,9 +730,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
 
   if (affectedRows) *affectedRows = nRow;
 
-  return code;
+_exit:
-
-_err:
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -89,8 +89,8 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
     pLoadInfo[i].blockIndex[0] = -1;
     pLoadInfo[i].blockIndex[1] = -1;
 
-    tBlockDataDestroy(&pLoadInfo[i].blockData[0], true);
+    tBlockDataDestroy(&pLoadInfo[i].blockData[0]);
-    tBlockDataDestroy(&pLoadInfo[i].blockData[1], true);
+    tBlockDataDestroy(&pLoadInfo[i].blockData[1]);
 
     taosArrayDestroy(pLoadInfo[i].aSttBlk);
   }
@@ -154,8 +154,8 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
 
   tsdbDebug("read last block, total load:%d, trigger by uid:%" PRIu64
             ", last file index:%d, last block index:%d, entry:%d, rows:%d, %p, elapsed time:%.2f ms, %s",
-            pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock->nRow, pBlock, el,
+            pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock->nRow,
-            idStr);
+            pBlock, el, idStr);
 
   pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk;
   tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr);

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -775,7 +775,10 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_
     } else {
       varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData);
       ASSERT(pColVal->value.nData <= pColInfoData->info.bytes);
+      if (pColVal->value.nData > 0) {  // pData may be null, if nData is 0
         memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData);
+      }
+
       colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false);
     }
   } else {
@@ -2539,6 +2542,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
       goto _err;
     }
 
+    // TODO: opt the perf of read del index
     code = tsdbReadDelIdx(pDelFReader, aDelIdx);
     if (code != TSDB_CODE_SUCCESS) {
       taosArrayDestroy(aDelIdx);
@@ -3879,7 +3883,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
   }
 
   taosMemoryFree(pSupInfo->buildBuf);
-  tBlockDataDestroy(&pReader->status.fileBlockData, true);
+  tBlockDataDestroy(&pReader->status.fileBlockData);
 
   cleanupDataBlockIterator(&pReader->status.blockIter);
 
@@ -4167,7 +4171,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
 
   int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid);
   if (code != TSDB_CODE_SUCCESS) {
-    tBlockDataDestroy(&pStatus->fileBlockData, 1);
+    tBlockDataDestroy(&pStatus->fileBlockData);
     terrno = code;
     return NULL;
   }

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -524,7 +524,7 @@ _exit:
     if (pReader) {
       taosArrayDestroy(pReader->aDelData);
       taosArrayDestroy(pReader->aDelIdx);
-      tBlockDataDestroy(&pReader->bData, 1);
+      tBlockDataDestroy(&pReader->bData);
       tsdbFSDestroy(&pReader->fs);
       taosMemoryFree(pReader);
     }
@@ -551,10 +551,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
       taosArrayDestroy(pIter->aSttBlk);
     }
 
-    tBlockDataDestroy(&pIter->bData, 1);
+    tBlockDataDestroy(&pIter->bData);
   }
 
-  tBlockDataDestroy(&pReader->bData, 1);
+  tBlockDataDestroy(&pReader->bData);
   tDestroyTSchema(pReader->skmTable.pTSchema);
 
   // del
@@ -1345,13 +1345,13 @@ _exit:
       if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW);
       if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData);
       if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR);
-      tBlockDataDestroy(&pWriter->dWriter.sData, 1);
+      tBlockDataDestroy(&pWriter->dWriter.sData);
-      tBlockDataDestroy(&pWriter->dWriter.bData, 1);
+      tBlockDataDestroy(&pWriter->dWriter.bData);
       if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk);
       if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx);
-      tBlockDataDestroy(&pWriter->dReader.bData, 1);
+      tBlockDataDestroy(&pWriter->dReader.bData);
       if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx);
-      tBlockDataDestroy(&pWriter->bData, 1);
+      tBlockDataDestroy(&pWriter->bData);
       tsdbFSDestroy(&pWriter->fs);
       taosMemoryFree(pWriter);
     }
@@ -1404,18 +1404,18 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
   // SNAP_DATA_TSDB
 
   // Writer
-  tBlockDataDestroy(&pWriter->dWriter.sData, 1);
+  tBlockDataDestroy(&pWriter->dWriter.sData);
-  tBlockDataDestroy(&pWriter->dWriter.bData, 1);
+  tBlockDataDestroy(&pWriter->dWriter.bData);
   taosArrayDestroy(pWriter->dWriter.aSttBlk);
   tMapDataClear(&pWriter->dWriter.mDataBlk);
   taosArrayDestroy(pWriter->dWriter.aBlockIdx);
 
   // Reader
-  tBlockDataDestroy(&pWriter->dReader.bData, 1);
+  tBlockDataDestroy(&pWriter->dReader.bData);
   tMapDataClear(&pWriter->dReader.mDataBlk);
   taosArrayDestroy(pWriter->dReader.aBlockIdx);
 
-  tBlockDataDestroy(&pWriter->bData, 1);
+  tBlockDataDestroy(&pWriter->bData);
   tDestroyTSchema(pWriter->skmTable.pTSchema);
 
   for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) {

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -23,7 +23,7 @@ void tMapDataReset(SMapData *pMapData) {
 }
 
 void tMapDataClear(SMapData *pMapData) {
-  tFree((uint8_t *)pMapData->aOffset);
+  tFree(pMapData->aOffset);
   tFree(pMapData->pData);
   pMapData->pData = NULL;
   pMapData->aOffset = NULL;
@@ -572,9 +572,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
 
   ASSERT(iCol > 0);
 
-  if (pRow->type == 0) {
+  if (pRow->type == TSDBROW_ROW_FMT) {
     tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal);
-  } else if (pRow->type == 1) {
+  } else if (pRow->type == TSDBROW_COL_FMT) {
     SColData *pColData;
 
     tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData);
@@ -606,11 +606,11 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) {
 // STSDBRowIter ======================================================
 void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
   pIter->pRow = pRow;
-  if (pRow->type == 0) {
+  if (pRow->type == TSDBROW_ROW_FMT) {
     ASSERT(pTSchema);
     pIter->pTSchema = pTSchema;
     pIter->i = 1;
-  } else if (pRow->type == 1) {
+  } else if (pRow->type == TSDBROW_COL_FMT) {
     pIter->pTSchema = NULL;
     pIter->i = 0;
   } else {
@@ -619,14 +619,14 @@ void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
 }
 
 SColVal *tsdbRowIterNext(STSDBRowIter *pIter) {
-  if (pIter->pRow->type == 0) {
+  if (pIter->pRow->type == TSDBROW_ROW_FMT) {
     if (pIter->i < pIter->pTSchema->numOfCols) {
       tRowGet(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal);
       pIter->i++;
 
       return &pIter->colVal;
     }
-  } else {
+  } else if (pIter->pRow->type == TSDBROW_COL_FMT) {
     if (pIter->i < pIter->pRow->pBlockData->nColData) {
       SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i);
 
@@ -635,6 +635,8 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) {
 
       return &pIter->colVal;
     }
+  } else {
+    ASSERT(0);
   }
 
   return NULL;
@@ -728,7 +730,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
         taosArraySet(pMerger->pArray, iCol, pColVal);
       }
     } else {
-      ASSERT(0);
+      ASSERT(0 && "dup versions not allowed");
     }
   }
 
@@ -898,7 +900,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _clear;
     }
-
     midx = (sidx + eidx) / 2;
 
     code = tsdbBuildDeleteSkyline(aDelData, sidx, midx, aSkyline1);
@@ -928,27 +929,49 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) {
   pBlockData->aVersion = NULL;
   pBlockData->aTSKEY = NULL;
   pBlockData->nColData = 0;
-  pBlockData->aColData = taosArrayInit(0, sizeof(SColData));
+  pBlockData->aColData = NULL;
-  if (pBlockData->aColData == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
-  }
 
 _exit:
   return code;
 }
 
-void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
+void tBlockDataDestroy(SBlockData *pBlockData) {
-  tFree((uint8_t *)pBlockData->aUid);
+  tFree(pBlockData->aUid);
-  tFree((uint8_t *)pBlockData->aVersion);
+  tFree(pBlockData->aVersion);
-  tFree((uint8_t *)pBlockData->aTSKEY);
+  tFree(pBlockData->aTSKEY);
-  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL);
+
-  pBlockData->aUid = NULL;
+  for (int32_t i = 0; i < pBlockData->nColData; i++) {
-  pBlockData->aVersion = NULL;
+    tColDataDestroy(&pBlockData->aColData[i]);
-  pBlockData->aTSKEY = NULL;
-  pBlockData->aColData = NULL;
   }
 
+  if (pBlockData->aColData) {
+    taosMemoryFree(pBlockData->aColData);
+    pBlockData->aColData = NULL;
+  }
+}
+
+static int32_t tBlockDataAdjustColData(SBlockData *pBlockData, int32_t nColData) {
+  int32_t code = 0;
+
+  if (pBlockData->nColData > nColData) {
+    for (int32_t i = nColData; i < pBlockData->nColData; i++) {
+      tColDataDestroy(&pBlockData->aColData[i]);
+    }
+  } else if (pBlockData->nColData < nColData) {
+    SColData *aColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SBlockData) * nColData);
+    if (aColData == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _exit;
+    }
+
+    pBlockData->aColData = aColData;
+    memset(&pBlockData->aColData[pBlockData->nColData], 0, sizeof(SBlockData) * (nColData - pBlockData->nColData));
+  }
+  pBlockData->nColData = nColData;
+
+_exit:
+  return code;
+}
 int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) {
   int32_t code = 0;
 
@@ -958,37 +981,35 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
   pBlockData->uid = pId->uid;
   pBlockData->nRow = 0;
 
-  pBlockData->nColData = 0;
   if (aCid) {
+    code = tBlockDataAdjustColData(pBlockData, nCid);
+    if (code) goto _exit;
+
     int32_t   iColumn = 1;
     STColumn *pTColumn = &pTSchema->columns[iColumn];
     for (int32_t iCid = 0; iCid < nCid; iCid++) {
-      while (pTColumn && pTColumn->colId < aCid[iCid]) {
+      ASSERT(pTColumn);
+      while (pTColumn->colId < aCid[iCid]) {
         iColumn++;
-        pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL;
+        ASSERT(iColumn < pTSchema->numOfCols);
+        pTColumn = &pTSchema->columns[iColumn];
       }
 
-      if (pTColumn == NULL) {
+      ASSERT(pTColumn->colId == aCid[iCid]);
-        break;
+      tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type,
-      } else if (pTColumn->colId == aCid[iCid]) {
+                   (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
-        SColData *pColData;
-        code = tBlockDataAddColData(pBlockData, &pColData);
-        if (code) goto _exit;
-        tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
 
       iColumn++;
       pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL;
     }
-    }
   } else {
-    for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) {
+    code = tBlockDataAdjustColData(pBlockData, pTSchema->numOfCols - 1);
-      STColumn *pTColumn = &pTSchema->columns[iColumn];
-
-      SColData *pColData;
-      code = tBlockDataAddColData(pBlockData, &pColData);
     if (code) goto _exit;
 
-      tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
+    for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
+      STColumn *pTColumn = &pTSchema->columns[iColData + 1];
+      tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type,
+                   (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
     }
   }
 
@@ -999,8 +1020,6 @@ _exit:
 void tBlockDataReset(SBlockData *pBlockData) {
   pBlockData->suid = 0;
   pBlockData->uid = 0;
-  pBlockData->nRow = 0;
-  pBlockData->nColData = 0;
 }
 
 void tBlockDataClear(SBlockData *pBlockData) {
@@ -1008,33 +1027,10 @@ void tBlockDataClear(SBlockData *pBlockData) {
 
   pBlockData->nRow = 0;
   for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
-    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
+    tColDataClear(tBlockDataGetColDataByIdx(pBlockData, iColData));
-    tColDataClear(pColData);
   }
 }
 
-int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData) {
-  int32_t   code = 0;
-  SColData *pColData = NULL;
-
-  if (pBlockData->nColData >= taosArrayGetSize(pBlockData->aColData)) {
-    if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
-    }
-  }
-  pColData = (SColData *)taosArrayGet(pBlockData->aColData, pBlockData->nColData);
-
-  pBlockData->nColData++;
-
-  *ppColData = pColData;
-  return code;
-
-_err:
-  *ppColData = NULL;
-  return code;
-}
-
 static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow) {
   int32_t code = 0;
 
@@ -1095,12 +1091,14 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
   pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow);
 
   SColVal cv = {0};
-  if (pRow->type == 0) {
+  if (pRow->type == TSDBROW_ROW_FMT) {
     code = tRowAppendToColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData);
     if (code) goto _err;
-  } else {
+  } else if (pRow->type == TSDBROW_COL_FMT) {
     code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow);
     if (code) goto _err;
+  } else {
+    ASSERT(0);
   }
   pBlockData->nRow++;
 
@@ -1110,133 +1108,13 @@ _err:
   return code;
 }
 
-int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) {
-  int32_t code = 0;
-
-  int32_t iColData = 0;
-  for (int32_t iColDataFrom = 0; iColDataFrom < pBlockDataFrom->nColData; iColDataFrom++) {
-    SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom);
-
-    while (true) {
-      SColData *pColData;
-      if (iColData < pBlockData->nColData) {
-        pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
-      } else {
-        pColData = NULL;
-      }
-
-      if (pColData == NULL || pColData->cid > pColDataFrom->cid) {
-        code = tBlockDataAddColData(pBlockData, &pColData);
-        if (code) goto _exit;
-
-        tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn);
-        for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
-          code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
-          if (code) goto _exit;
-        }
-
-        iColData++;
-        break;
-      } else if (pColData->cid == pColDataFrom->cid) {
-        iColData++;
-        break;
-      } else {
-        iColData++;
-      }
-    }
-  }
-
-_exit:
-  return code;
-}
-
-int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) {
-  int32_t code = 0;
-
-  ASSERT(pBlockData->suid == pBlockData1->suid);
-  ASSERT(pBlockData->uid == pBlockData1->uid);
-  ASSERT(pBlockData1->nRow > 0);
-  ASSERT(pBlockData2->nRow > 0);
-
-  tBlockDataClear(pBlockData);
-
-  TSDBROW  row1 = tsdbRowFromBlockData(pBlockData1, 0);
-  TSDBROW  row2 = tsdbRowFromBlockData(pBlockData2, 0);
-  TSDBROW *pRow1 = &row1;
-  TSDBROW *pRow2 = &row2;
-
-  while (pRow1 && pRow2) {
-    int32_t c = tsdbRowCmprFn(pRow1, pRow2);
-
-    if (c < 0) {
-      code = tBlockDataAppendRow(pBlockData, pRow1, NULL,
-                                 pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]);
-      if (code) goto _exit;
-
-      pRow1->iRow++;
-      if (pRow1->iRow < pBlockData1->nRow) {
-        *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow);
-      } else {
-        pRow1 = NULL;
-      }
-    } else if (c > 0) {
-      code = tBlockDataAppendRow(pBlockData, pRow2, NULL,
-                                 pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]);
-      if (code) goto _exit;
-
-      pRow2->iRow++;
-      if (pRow2->iRow < pBlockData2->nRow) {
-        *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow);
-      } else {
-        pRow2 = NULL;
-      }
-    } else {
-      ASSERT(0);
-    }
-  }
-
-  while (pRow1) {
-    code = tBlockDataAppendRow(pBlockData, pRow1, NULL,
-                               pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]);
-    if (code) goto _exit;
-
-    pRow1->iRow++;
-    if (pRow1->iRow < pBlockData1->nRow) {
-      *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow);
-    } else {
-      pRow1 = NULL;
-    }
-  }
-
-  while (pRow2) {
-    code = tBlockDataAppendRow(pBlockData, pRow2, NULL,
-                               pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]);
-    if (code) goto _exit;
-
-    pRow2->iRow++;
-    if (pRow2->iRow < pBlockData2->nRow) {
-      *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow);
-    } else {
-      pRow2 = NULL;
-    }
-  }
-
-_exit:
-  return code;
-}
-
-SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) {
-  ASSERT(idx >= 0 && idx < pBlockData->nColData);
-  return (SColData *)taosArrayGet(pBlockData->aColData, idx);
-}
-
 void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) {
   ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID);
   int32_t lidx = 0;
   int32_t ridx = pBlockData->nColData - 1;
 
   while (lidx <= ridx) {
-    int32_t   midx = (lidx + ridx) / 2;
+    int32_t   midx = (lidx + ridx) >> 2;
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx);
     int32_t   c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1);
 
@@ -1383,16 +1261,26 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
   // loop to decode each column data
   if (hdr.szBlkCol == 0) goto _exit;
 
+  int32_t nColData = 0;
   int32_t nt = 0;
   while (nt < hdr.szBlkCol) {
     SBlockCol blockCol = {0};
     nt += tGetBlockCol(pIn + n + nt, &blockCol);
-    ASSERT(nt <= hdr.szBlkCol);
+    ++nColData;
+  }
+  ASSERT(nt == hdr.szBlkCol);
 
-    SColData *pColData;
+  code = tBlockDataAdjustColData(pBlockData, nColData);
-    code = tBlockDataAddColData(pBlockData, &pColData);
   if (code) goto _exit;
 
+  nt = 0;
+  int32_t iColData = 0;
+  while (nt < hdr.szBlkCol) {
+    SBlockCol blockCol = {0};
+    nt += tGetBlockCol(pIn + n + nt, &blockCol);
+
+    SColData *pColData = &pBlockData->aColData[iColData++];
+
     tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn);
     if (blockCol.flag == HAS_NULL) {
       for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) {

source/dnode/vnode/src/vnd/vnodeBufPool.c

@@ -123,7 +123,7 @@ void vnodeBufPoolReset(SVBufPool *pPool) {
   pPool->ptr = pPool->node.data;
 }
 
-void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
+void *vnodeBufPoolMalloc(SVBufPool *pPool, int32_t size) {
   SVBufPoolNode *pNode;
   void          *p = NULL;
   ASSERT(pPool != NULL);
@@ -176,6 +176,9 @@ void vnodeBufPoolRef(SVBufPool *pPool) {
 }
 
 void vnodeBufPoolUnRef(SVBufPool *pPool) {
+  if (pPool == NULL) {
+    return;
+  }
   int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1);
   if (nRef == 0) {
     SVnode *pVnode = pPool->pVnode;

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -209,8 +209,8 @@ int vnodeCommit(SVnode *pVnode) {
   SVnodeInfo info = {0};
   char       dir[TSDB_FILENAME_LEN];
 
-  vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID,
+  vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode),
-        pVnode->state.applied);
+        pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm);
 
   // persist wal before starting
   if (walPersist(pVnode->pWal) < 0) {

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -144,9 +144,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->config = info.config;
   pVnode->state.committed = info.state.committed;
   pVnode->state.commitTerm = info.state.commitTerm;
-  pVnode->state.applied = info.state.committed;
   pVnode->state.commitID = info.state.commitID;
-  pVnode->state.commitTerm = info.state.commitTerm;
+  pVnode->state.applied = info.state.committed;
+  pVnode->state.applyTerm = info.state.commitTerm;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
   taosThreadMutexInit(&pVnode->lock, NULL);
@@ -269,10 +269,7 @@ void vnodeClose(SVnode *pVnode) {
 }
 
 // start the sync timer after the queue is ready
-int32_t vnodeStart(SVnode *pVnode) {
+int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); }
-  vnodeSyncStart(pVnode);
-  return 0;
-}
 
 void vnodeStop(SVnode *pVnode) {}
 

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -380,6 +380,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
   pLoad->vgId = TD_VID(pVnode);
   pLoad->syncState = state.state;
   pLoad->syncRestore = state.restored;
+  pLoad->syncCanRead = state.canRead;
   pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
   pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
   pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -187,8 +187,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
     return -1;
   }
 
+  if (version <= pVnode->state.applied) {
+    vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version,
+           pVnode->state.applied);
+    pRsp->info.handle = NULL;
+    return -1;
+  }
+
   vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
          version);
+  ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm);
 
   pVnode->state.applied = version;
   pVnode->state.applyTerm = pMsg->info.conn.applyTerm;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -295,36 +295,31 @@ static int32_t vnodeSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot)
   return 0;
 }
 
-static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
+static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
   SVnode *pVnode = pFsm->data;
-
+  pMsg->info.conn.applyIndex = pMeta->index;
-  SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen};
+  pMsg->info.conn.applyTerm = pMeta->term;
-  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
-  memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
-  rpcMsg.info = pMsg->info;
-  rpcMsg.info.conn.applyIndex = pMeta->index;
-  rpcMsg.info.conn.applyTerm = pMeta->term;
 
   const STraceId *trace = &pMsg->info.traceId;
   vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64
           ", weak:%d, code:%d, state:%d %s, type:%s",
-          pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code,
+          pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code,
           pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType));
 
-  tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
+  tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg);
 }
 
-static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
+static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
   vnodeSyncApplyMsg(pFsm, pMsg, pMeta);
 }
 
-static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
+static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
   if (pMeta->isWeak == 1) {
     vnodeSyncApplyMsg(pFsm, pMsg, pMeta);
   }
 }
 
-static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
+static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
   SVnode *pVnode = pFsm->data;
   vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s",
          pVnode->config.vgId, pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state),
@@ -404,7 +399,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm) {
   walApplyVer(pVnode->pWal, pVnode->state.applied);
 
   pVnode->restored = true;
-  vDebug("vgId:%d, sync restore finished", pVnode->config.vgId);
+  vInfo("vgId:%d, sync restore finished", pVnode->config.vgId);
 }
 
 static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
@@ -506,15 +501,29 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
   return 0;
 }
 
-void vnodeSyncStart(SVnode *pVnode) {
+int32_t vnodeSyncStart(SVnode *pVnode) {
   vInfo("vgId:%d, start sync", pVnode->config.vgId);
-  syncStart(pVnode->sync);
+  if (syncStart(pVnode->sync) < 0) {
+    vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, terrstr());
+    return -1;
+  }
+  return 0;
 }
 
 void vnodeSyncPreClose(SVnode *pVnode) {
   vInfo("vgId:%d, pre close sync", pVnode->config.vgId);
   syncLeaderTransfer(pVnode->sync);
   syncPreStop(pVnode->sync);
+#if 0
+  while (syncSnapshotRecving(pVnode->sync)) {
+    vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId);
+    taosMsleep(300);
+  }
+  while (syncSnapshotSending(pVnode->sync)) {
+    vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId);
+    taosMsleep(300);
+  }
+#endif
   taosThreadMutexLock(&pVnode->lock);
   if (pVnode->blocked) {
     vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);
@@ -543,12 +552,12 @@ bool vnodeIsLeader(SVnode *pVnode) {
     } else {
       terrno = TSDB_CODE_APP_NOT_READY;
     }
-    vDebug("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored);
+    vInfo("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored);
     return false;
   }
 
   if (!pVnode->restored) {
-    vDebug("vgId:%d, vnode not restored", pVnode->config.vgId);
+    vInfo("vgId:%d, vnode not restored", pVnode->config.vgId);
     terrno = TSDB_CODE_APP_NOT_READY;
     return false;
   }

source/libs/executor/inc/executil.h

@@ -161,4 +161,6 @@ int32_t convertFillType(int32_t mode);
 int32_t resultrowComparAsc(const void* p1, const void* p2);
 int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
 
+void printDataBlock(SSDataBlock* pBlock, const char* flag);
+
 #endif  // TDENGINE_QUERYUTIL_H

source/libs/executor/inc/executorimpl.h

@@ -536,25 +536,9 @@ typedef struct SStreamIntervalOperatorInfo {
   SArray*            pChildren;
   SStreamState*      pState;
   SWinKey            delKey;
+  uint64_t           numOfDatapack;
 } SStreamIntervalOperatorInfo;
 
-typedef struct SFillOperatorInfo {
-  struct SFillInfo* pFillInfo;
-  SSDataBlock*      pRes;
-  SSDataBlock*      pFinalRes;
-  int64_t           totalInputRows;
-  void**            p;
-  SSDataBlock*      existNewGroupBlock;
-  STimeWindow       win;
-  SColMatchInfo     matchInfo;
-  int32_t           primaryTsCol;
-  int32_t           primarySrcSlotId;
-  uint64_t          curGroupId;  // current handled group id
-  SExprInfo*        pExprInfo;
-  int32_t           numOfExpr;
-  SExprSupp         noFillExprSupp;
-} SFillOperatorInfo;
-
 typedef struct SDataGroupInfo {
   uint64_t groupId;
   int64_t  numOfRows;
@@ -806,8 +790,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
 int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
 int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
 
-void    printTaskExecCostInLog(SExecTaskInfo* pTaskInfo);
-
 int32_t getMaximumIdleDurationSec();
 
 STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
@@ -825,7 +807,6 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
 bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup);
 void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
                                       uint64_t* pGp, void* pTbName);
-void printDataBlock(SSDataBlock* pBlock, const char* flag);
 uint64_t    calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
 void        calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock);
 

source/libs/executor/inc/tfill.h

@@ -25,6 +25,8 @@ extern "C" {
 #include "tcommon.h"
 #include "tsimplehash.h"
 
+#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId)
+
 struct SSDataBlock;
 
 typedef struct SFillColInfo {
@@ -116,7 +118,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t
 void          taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
 void          taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp);
 void          taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
-struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr,
+SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr,
                                 int32_t numOfNotFillCols, const struct SNodeListNode* val);
 bool          taosFillHasMoreResults(struct SFillInfo* pFillInfo);
 
@@ -128,6 +130,8 @@ void*   taosDestroyFillInfo(struct SFillInfo* pFillInfo);
 int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity);
 int64_t getFillInfoStart(struct SFillInfo* pFillInfo);
 
+bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData,
+                                     int32_t rowIndex);
 #ifdef __cplusplus
 }
 #endif

source/libs/executor/src/exchangeoperator.c

@@ -15,26 +15,15 @@
 
 #include "filter.h"
 #include "function.h"
-#include "functionMgt.h"
 #include "os.h"
-#include "querynodes.h"
-#include "tfill.h"
 #include "tname.h"
 #include "tref.h"
-
 #include "tdatablock.h"
-#include "tglobal.h"
 #include "tmsg.h"
-#include "tsort.h"
-#include "ttime.h"
-
 #include "executorimpl.h"
 #include "index.h"
 #include "query.h"
-#include "tcompare.h"
 #include "thash.h"
-#include "ttypes.h"
-#include "vnode.h"
 
 typedef struct SFetchRspHandleWrapper {
   uint32_t exchangeId;

source/libs/executor/src/executil.c

@@ -2002,3 +2002,13 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
 
   return TSDB_CODE_SUCCESS;
 }
+
+void printDataBlock(SSDataBlock* pBlock, const char* flag) {
+  if (!pBlock || pBlock->info.rows == 0) {
+    qDebug("===stream===printDataBlock: Block is Null or Empty");
+    return;
+  }
+  char* pBuf = NULL;
+  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
+  taosMemoryFree(pBuf);
+}

source/libs/executor/src/executor.c

@@ -704,6 +704,20 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
   return TSDB_CODE_SUCCESS;
 }
 
+static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
+  STaskCostInfo* pSummary = &pTaskInfo->cost;
+
+  SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
+  if (pSummary->pRecoder != NULL) {
+    qDebug(
+        "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, "
+        "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64,
+        GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime,
+        pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows,
+        pRecorder->totalCheckedRows);
+  }
+}
+
 void qDestroyTask(qTaskInfo_t qTaskHandle) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle;
   if (pTaskInfo == NULL) {

source/libs/executor/src/executorimpl.c

@@ -91,7 +91,6 @@ static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock*
 
 static void releaseQueryBuf(size_t numOfTables);
 
-static void    destroyFillOperatorInfo(void* param);
 static void    destroyAggOperatorInfo(void* param);
 static void    initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
 static void    doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
@@ -1157,20 +1156,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
   }
 }
 
-void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
-  STaskCostInfo* pSummary = &pTaskInfo->cost;
-
-  SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
-  if (pSummary->pRecoder != NULL) {
-    qDebug(
-        "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, "
-        "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64,
-        GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime,
-        pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows,
-        pRecorder->totalCheckedRows);
-  }
-}
-
 // void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) {
 //   STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
 //
@@ -1603,179 +1588,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
   return (rows == 0) ? NULL : pInfo->pRes;
 }
 
-static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
-                                               SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
-  pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
-  SSDataBlock* pResBlock = pInfo->pFinalRes;
-
-  int32_t order = TSDB_ORDER_ASC;
-  int32_t scanFlag = MAIN_SCAN;
-  getTableScanInfo(pOperator, &order, &scanFlag);
-
-  int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey;
-  taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));
-
-  blockDataCleanup(pInfo->pRes);
-  doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag);
-
-  taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey);
-  taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
-
-  int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows;
-  taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows);
-
-  pInfo->curGroupId = pInfo->existNewGroupBlock->info.id.groupId;
-  pInfo->existNewGroupBlock = NULL;
-}
-
-static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
-                                            SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
-  if (taosFillHasMoreResults(pInfo->pFillInfo)) {
-    int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows;
-    taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows);
-    pInfo->pRes->info.id.groupId = pInfo->curGroupId;
-    return;
-  }
-
-  // handle the cached new group data block
-  if (pInfo->existNewGroupBlock) {
-    doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo);
-  }
-}
-
-static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) {
-  SFillOperatorInfo* pInfo = pOperator->info;
-  SExprSupp*         pSup = &pOperator->exprSupp;
-  setInputDataBlock(pSup, pBlock, order, scanFlag, false);
-  projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, NULL);
-
-  // reset the row value before applying the no-fill functions to the input data block, which is "pBlock" in this case.
-  pInfo->pRes->info.rows = 0;
-  SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp;
-  setInputDataBlock(pNoFillSupp, pBlock, order, scanFlag, false);
-
-  projectApplyFunctions(pNoFillSupp->pExprInfo, pInfo->pRes, pBlock, pNoFillSupp->pCtx, pNoFillSupp->numOfExprs, NULL);
-  pInfo->pRes->info.id.groupId = pBlock->info.id.groupId;
-}
-
-static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
-  SFillOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*     pTaskInfo = pOperator->pTaskInfo;
-
-  SResultInfo* pResultInfo = &pOperator->resultInfo;
-  SSDataBlock* pResBlock = pInfo->pFinalRes;
-
-  blockDataCleanup(pResBlock);
-
-  int32_t order = TSDB_ORDER_ASC;
-  int32_t scanFlag = MAIN_SCAN;
-  getTableScanInfo(pOperator, &order, &scanFlag);
-
-  doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
-  if (pResBlock->info.rows > 0) {
-    pResBlock->info.id.groupId = pInfo->curGroupId;
-    return pResBlock;
-  }
-
-  SOperatorInfo* pDownstream = pOperator->pDownstream[0];
-  while (1) {
-    SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream);
-    if (pBlock == NULL) {
-      if (pInfo->totalInputRows == 0) {
-        setOperatorCompleted(pOperator);
-        return NULL;
-      }
-
-      taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
-    } else {
-      blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId);
-
-      blockDataCleanup(pInfo->pRes);
-      blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
-      blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
-      doApplyScalarCalculation(pOperator, pBlock, order, scanFlag);
-
-      if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) {
-        pInfo->curGroupId = pInfo->pRes->info.id.groupId;  // the first data block
-        pInfo->totalInputRows += pInfo->pRes->info.rows;
-
-        if (order == pInfo->pFillInfo->order) {
-          taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey);
-        } else {
-          taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey);
-        }
-        taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
-      } else if (pInfo->curGroupId != pBlock->info.id.groupId) {  // the new group data block
-        pInfo->existNewGroupBlock = pBlock;
-
-        // Fill the previous group data block, before handle the data block of new group.
-        // Close the fill operation for previous group data block
-        taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
-      }
-    }
-
-    int32_t numOfResultRows = pOperator->resultInfo.capacity - pResBlock->info.rows;
-    taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows);
-
-    // current group has no more result to return
-    if (pResBlock->info.rows > 0) {
-      // 1. The result in current group not reach the threshold of output result, continue
-      // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately
-      if (pResBlock->info.rows > pResultInfo->threshold || pBlock == NULL || pInfo->existNewGroupBlock != NULL) {
-        pResBlock->info.id.groupId = pInfo->curGroupId;
-        return pResBlock;
-      }
-
-      doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
-      if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) {
-        pResBlock->info.id.groupId = pInfo->curGroupId;
-        return pResBlock;
-      }
-    } else if (pInfo->existNewGroupBlock) {  // try next group
-      assert(pBlock != NULL);
-
-      blockDataCleanup(pResBlock);
-
-      doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo);
-      if (pResBlock->info.rows > pResultInfo->threshold) {
-        pResBlock->info.id.groupId = pInfo->curGroupId;
-        return pResBlock;
-      }
-    } else {
-      return NULL;
-    }
-  }
-}
-
-static SSDataBlock* doFill(SOperatorInfo* pOperator) {
-  SFillOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*     pTaskInfo = pOperator->pTaskInfo;
-
-  if (pOperator->status == OP_EXEC_DONE) {
-    return NULL;
-  }
-
-  SSDataBlock* fillResult = NULL;
-  while (true) {
-    fillResult = doFillImpl(pOperator);
-    if (fillResult == NULL) {
-      setOperatorCompleted(pOperator);
-      break;
-    }
-
-    doFilter(fillResult, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
-    if (fillResult->info.rows > 0) {
-      break;
-    }
-  }
-
-  if (fillResult != NULL) {
-    pOperator->resultInfo.totalRows += fillResult->info.rows;
-  }
-
-  return fillResult;
-}
-
 void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
   for (int32_t i = 0; i < numOfExprs; ++i) {
     SExprInfo* pExprInfo = &pExpr[i];
@@ -2045,167 +1857,6 @@ void destroyAggOperatorInfo(void* param) {
   taosMemoryFreeClear(param);
 }
 
-void destroyFillOperatorInfo(void* param) {
-  SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param;
-  pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
-  pInfo->pRes = blockDataDestroy(pInfo->pRes);
-  pInfo->pFinalRes = blockDataDestroy(pInfo->pFinalRes);
-
-  cleanupExprSupp(&pInfo->noFillExprSupp);
-
-  taosMemoryFreeClear(pInfo->p);
-  taosArrayDestroy(pInfo->matchInfo.pList);
-  taosMemoryFreeClear(param);
-}
-
-static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr,
-                            int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity,
-                            const char* id, SInterval* pInterval, int32_t fillType, int32_t order) {
-  SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode);
-
-  int64_t     startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey;
-  STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey);
-  w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order);
-
-  pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo,
-                                        pInfo->primaryTsCol, order, id);
-
-  if (order == TSDB_ORDER_ASC) {
-    pInfo->win.skey = win.skey;
-    pInfo->win.ekey = win.ekey;
-  } else {
-    pInfo->win.skey = win.ekey;
-    pInfo->win.ekey = win.skey;
-  }
-  pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-
-  if (pInfo->pFillInfo == NULL || pInfo->p == NULL) {
-    taosMemoryFree(pInfo->pFillInfo);
-    taosMemoryFree(pInfo->p);
-    return TSDB_CODE_OUT_OF_MEMORY;
-  } else {
-    return TSDB_CODE_SUCCESS;
-  }
-}
-
-static bool isWstartColumnExist(SFillOperatorInfo* pInfo) {
-  if (pInfo->noFillExprSupp.numOfExprs == 0) {
-    return false;
-  }
-
-  for (int32_t i = 0; i < pInfo->noFillExprSupp.numOfExprs; ++i) {
-    SExprInfo* exprInfo = pInfo->noFillExprSupp.pExprInfo + i;
-    if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN && exprInfo->base.numOfParams == 1 &&
-        exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) {
-      return true;
-    }
-  }
-  return false;
-}
-
-static int32_t createPrimaryTsExprIfNeeded(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode, SExprSupp* pExprSupp,
-                                           const char* idStr) {
-  bool wstartExist = isWstartColumnExist(pInfo);
-
-  if (wstartExist == false) {
-    if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) {
-      qError("pWStartTs of fill physical node is not a target node, %s", idStr);
-      return TSDB_CODE_QRY_SYS_ERROR;
-    }
-
-    SExprInfo* pExpr = taosMemoryRealloc(pExprSupp->pExprInfo, (pExprSupp->numOfExprs + 1) * sizeof(SExprInfo));
-    if (pExpr == NULL) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    createExprFromTargetNode(&pExpr[pExprSupp->numOfExprs], (STargetNode*)pPhyFillNode->pWStartTs);
-    pExprSupp->numOfExprs += 1;
-    pExprSupp->pExprInfo = pExpr;
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode,
-                                      SExecTaskInfo* pTaskInfo) {
-  SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo));
-  SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
-  if (pInfo == NULL || pOperator == NULL) {
-    goto _error;
-  }
-
-  pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
-  SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr);
-  pOperator->exprSupp.pExprInfo = pExprInfo;
-
-  SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp;
-  pNoFillSupp->pExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pNoFillSupp->numOfExprs);
-  int32_t code = createPrimaryTsExprIfNeeded(pInfo, pPhyFillNode, pNoFillSupp, pTaskInfo->id.str);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  SInterval* pInterval =
-      QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
-          ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
-          : &((SIntervalAggOperatorInfo*)downstream->info)->interval;
-
-  int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
-  int32_t type = convertFillType(pPhyFillNode->mode);
-
-  SResultInfo* pResultInfo = &pOperator->resultInfo;
-
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
-  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
-  code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId;
-  pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
-
-  int32_t numOfOutputCols = 0;
-  code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols,
-                             COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
-
-  code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs,
-                      (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity,
-                      pTaskInfo->id.str, pInterval, type, order);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false);
-  blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity);
-
-  code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo);
-  pOperator->exprSupp.numOfExprs = pInfo->numOfExpr;
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL);
-
-  code = appendDownstream(pOperator, &downstream, 1);
-  return pOperator;
-
-_error:
-  if (pInfo != NULL) {
-    destroyFillOperatorInfo(pInfo);
-  }
-
-  pTaskInfo->code = code;
-  taosMemoryFreeClear(pOperator);
-  return NULL;
-}
-
 static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) {
   SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
   if (pTaskInfo == NULL) {

source/libs/executor/src/scanoperator.c

@@ -1052,6 +1052,9 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
 }
 
 static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
+  if (pBlock->info.rows == 0) {
+    return false;
+  }
   if ((*pRowIndex) == pBlock->info.rows) {
     return false;
   }
@@ -1110,7 +1113,7 @@ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t
     if (hasGroup) {
       (*pRowIndex) += 1;
     } else {
-      while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] < endWin.ekey)) {
+      while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] <= endWin.ekey)) {
         (*pRowIndex) += 1;
         if ((*pRowIndex) == pDataBlockInfo->rows) {
           break;
@@ -1184,10 +1187,10 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
 }
 
 static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
+  blockDataCleanup(pDestBlock);
   if (pSrcBlock->info.rows == 0) {
     return TSDB_CODE_SUCCESS;
   }
-  blockDataCleanup(pDestBlock);
   int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
@@ -1336,30 +1339,6 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
   return code;
 }
 
-static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* pResBlock) {
-  if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return;
-  if (pBlock == NULL || pBlock->info.rows == 0) return;
-
-  SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0);
-  ASSERT(pSrcBlock->info.rows == 1);
-
-  blockDataEnsureCapacity(pResBlock, 1);
-
-  projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL);
-  ASSERT(pResBlock->info.rows == 1);
-
-  // build tagArray
-  /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/
-  /*STagVal tagVal = {*/
-  /*.cid = 0,*/
-  /*.type = 0,*/
-  /*};*/
-  // build STag
-  // set STag
-
-  blockDataDestroy(pSrcBlock);
-}
-
 void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
   SExprSupp*    pTbNameCalSup = &pInfo->tbnameCalSup;
   SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState;
@@ -1837,6 +1816,12 @@ FETCH_NEXT_BLOCK:
         }
         setBlockGroupIdByUid(pInfo, pDelBlock);
         printDataBlock(pDelBlock, "stream scan delete recv filtered");
+        if (pDelBlock->info.rows == 0) {
+          if (pInfo->tqReader) {
+            blockDataDestroy(pDelBlock);
+          }
+          goto FETCH_NEXT_BLOCK;
+        }
         if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) {
           generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes);
           pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;

source/libs/executor/src/timesliceoperator.c

@@ -467,11 +467,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
         // add current row if timestamp match
         if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
           addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
-          doKeepPrevRows(pSliceInfo, pBlock, i);
 
           pSliceInfo->current =
               taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
         }
+        doKeepPrevRows(pSliceInfo, pBlock, i);
 
         if (pSliceInfo->current > pSliceInfo->win.ekey) {
           setOperatorCompleted(pOperator);

source/libs/executor/src/timewindowoperator.c

@@ -677,16 +677,6 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
   }
 }
 
-void printDataBlock(SSDataBlock* pBlock, const char* flag) {
-  if (!pBlock || pBlock->info.rows == 0) {
-    qDebug("===stream===printDataBlock: Block is Null or Empty");
-    return;
-  }
-  char* pBuf = NULL;
-  qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
-  taosMemoryFree(pBuf);
-}
-
 typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);
 
 int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
@@ -2526,9 +2516,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
     if (pBlock == NULL) {
       pOperator->status = OP_RES_TO_RETURN;
-      qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
+      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64 , IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
+      pInfo->numOfDatapack = 0;
       break;
     }
+    pInfo->numOfDatapack++;
     printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
 
     ASSERT(pBlock->info.type != STREAM_INVERT);
@@ -2744,6 +2736,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
   pInfo->delKey.ts = INT64_MAX;
   pInfo->delKey.groupId = 0;
+  pInfo->numOfDatapack = 0;
 
   pOperator->operatorType = pPhyNode->type;
   pOperator->blocking = true;
@@ -3854,7 +3847,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
   SStreamStateAggOperatorInfo* pInfo = pOperator->info;
   int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
-  int64_t                      groupId = pSDataBlock->info.id.groupId;
+  uint64_t                     groupId = pSDataBlock->info.id.groupId;
   int64_t                      code = TSDB_CODE_SUCCESS;
   TSKEY*                       tsCols = NULL;
   SResultRow*                  pResult = NULL;
@@ -4710,8 +4703,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   while (1) {
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
     if (pBlock == NULL) {
+      qDebug("===stream===return data:single interval. recv datablock num:%" PRIu64, pInfo->numOfDatapack);
+      pInfo->numOfDatapack = 0;
       break;
     }
+    pInfo->numOfDatapack++;
     printDataBlock(pBlock, "single interval recv");
 
     if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
@@ -4861,6 +4857,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pChildren = NULL;
   pInfo->delKey.ts = INT64_MAX;
   pInfo->delKey.groupId = 0;
+  pInfo->numOfDatapack = 0;
 
   setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
                   pInfo, pTaskInfo);

source/libs/function/src/detail/tavgfunction.c

@@ -133,6 +133,14 @@ static void i8VectorSumAVX2(const int8_t* plist, int32_t numOfRows, int32_t type
       sum = _mm256_add_epi64(sum, extVal);
       p += width;
     }
+
+    // let sum up the final results
+    const int64_t* q = (const int64_t*)∑
+    pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+
+    for (int32_t j = 0; j < remainder; ++j) {
+      pRes->sum.isum += plist[j + rounds * width];
+    }
   } else {
     const uint8_t* p = (const uint8_t*)plist;
 
@@ -142,16 +150,16 @@ static void i8VectorSumAVX2(const int8_t* plist, int32_t numOfRows, int32_t type
       sum = _mm256_add_epi64(sum, extVal);
       p += width;
     }
-  }
 
     // let sum up the final results
-  const int64_t* q = (const int64_t*)&sum;
+    const uint64_t* q = (const uint64_t*)&sum;
-  pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+    pRes->sum.usum += q[0] + q[1] + q[2] + q[3];
 
-  int32_t startIndex = rounds * width;
     for (int32_t j = 0; j < remainder; ++j) {
-    pRes->sum.isum += plist[j + startIndex];
+      pRes->sum.usum += (uint8_t)plist[j + rounds * width];
     }
+  }
+
 #endif
 }
 
@@ -176,8 +184,16 @@ static void i16VectorSumAVX2(const int16_t* plist, int32_t numOfRows, int32_t ty
       sum = _mm256_add_epi64(sum, extVal);
       p += width;
     }
+
+    // let sum up the final results
+    const int64_t* q = (const int64_t*)&sum;
+    pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+
+    for (int32_t j = 0; j < remainder; ++j) {
+      pRes->sum.isum += plist[j + rounds * width];
+    }
   } else {
-    const uint8_t* p = (const uint8_t*)plist;
+    const uint16_t* p = (const uint16_t*)plist;
 
     for(int32_t i = 0; i < rounds; ++i) {
       __m128i val = _mm_lddqu_si128((__m128i*)p);
@@ -185,16 +201,16 @@ static void i16VectorSumAVX2(const int16_t* plist, int32_t numOfRows, int32_t ty
       sum = _mm256_add_epi64(sum, extVal);
       p += width;
     }
-  }
 
     // let sum up the final results
-  const int64_t* q = (const int64_t*)&sum;
+    const uint64_t* q = (const uint64_t*)&sum;
-  pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+    pRes->sum.usum += q[0] + q[1] + q[2] + q[3];
 
-  int32_t startIndex = rounds * width;
     for (int32_t j = 0; j < remainder; ++j) {
-    pRes->sum.isum += plist[j + startIndex];
+      pRes->sum.usum += (uint16_t)plist[j + rounds * width];
     }
+  }
+
 #endif
 }
 
@@ -219,6 +235,14 @@ static void i32VectorSumAVX2(const int32_t* plist, int32_t numOfRows, int32_t ty
       sum = _mm256_add_epi64(sum, extVal);
       p += width;
     }
+
+    // let sum up the final results
+    const int64_t* q = (const int64_t*)&sum;
+    pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+
+    for (int32_t j = 0; j < remainder; ++j) {
+      pRes->sum.isum += plist[j + rounds * width];
+    }
   } else {
     const uint32_t* p = (const uint32_t*)plist;
 
@@ -228,16 +252,16 @@ static void i32VectorSumAVX2(const int32_t* plist, int32_t numOfRows, int32_t ty
       sum = _mm256_add_epi64(sum, extVal);
       p += width;
     }
-  }
 
     // let sum up the final results
-  const int64_t* q = (const int64_t*)&sum;
+    const uint64_t* q = (const uint64_t*)&sum;
-  pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+    pRes->sum.usum += q[0] + q[1] + q[2] + q[3];
 
-  int32_t startIndex = rounds * width;
     for (int32_t j = 0; j < remainder; ++j) {
-    pRes->sum.isum += plist[j + startIndex];
+      pRes->sum.usum += (uint32_t)plist[j + rounds * width];
     }
+  }
+
 #endif
 }
 
@@ -262,13 +286,22 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p
   }
 
   // let sum up the final results
+  if (type == TSDB_DATA_TYPE_BIGINT) {
     const int64_t* q = (const int64_t*)&sum;
     pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
 
-  int32_t startIndex = rounds * width;
     for (int32_t j = 0; j < remainder; ++j) {
-    pRes->sum.isum += plist[j + startIndex];
+      pRes->sum.isum += plist[j + rounds * width];
     }
+  } else {
+    const uint64_t* q = (const uint64_t*)&sum;
+    pRes->sum.usum += q[0] + q[1] + q[2] + q[3];
+
+    for (int32_t j = 0; j < remainder; ++j) {
+      pRes->sum.usum += (uint64_t)plist[j + rounds * width];
+    }
+  }
+
 #endif
 }
 
@@ -502,7 +535,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
           i8VectorSumAVX2(plist, numOfRows, type, pAvgRes);
         } else {
           for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
-            pAvgRes->sum.usum += plist[i];
+            if (type == TSDB_DATA_TYPE_TINYINT) {
+              pAvgRes->sum.isum += plist[i];
+            } else {
+              pAvgRes->sum.usum += (uint8_t)plist[i];
+            }
           }
         }
         break;
@@ -517,7 +554,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
           i16VectorSumAVX2(plist, numOfRows, type, pAvgRes);
         } else {
           for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+            if (type == TSDB_DATA_TYPE_SMALLINT) {
               pAvgRes->sum.isum += plist[i];
+            } else {
+              pAvgRes->sum.usum += (uint16_t)plist[i];
+            }
           }
         }
         break;
@@ -532,7 +573,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
           i32VectorSumAVX2(plist, numOfRows, type, pAvgRes);
         } else {
           for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+            if (type == TSDB_DATA_TYPE_INT) {
               pAvgRes->sum.isum += plist[i];
+            } else {
+              pAvgRes->sum.usum += (uint32_t)plist[i];
+            }
           }
         }
         break;
@@ -547,7 +592,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
           i64VectorSumAVX2(plist, numOfRows, pAvgRes);
         } else {
           for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+            if (type == TSDB_DATA_TYPE_BIGINT) {
               pAvgRes->sum.isum += plist[i];
+            } else {
+              pAvgRes->sum.isum += (uint64_t)plist[i];
+            }
           }
         }
         break;

source/libs/index/src/indexFilter.c

@@ -231,8 +231,10 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
       SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
       param->colId = -1;
       param->colValType = (uint8_t)(vn->node.resType.type);
-      if (strlen(vn->literal) <= sizeof(param->colName)) {
+      if (vn->literal != NULL && strlen(vn->literal) <= sizeof(param->colName)) {
         memcpy(param->colName, vn->literal, strlen(vn->literal));
+      } else {
+        param->status = SFLT_NOT_INDEX;
       }
       break;
     }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -424,6 +424,7 @@ SNode* nodesMakeNode(ENodeType type) {
     case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
     case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
     case QUERY_NODE_SHOW_TAGS_STMT:
+    case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
       return makeNode(type, sizeof(SShowStmt));
     case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
       return makeNode(type, sizeof(SShowTableTagsStmt));
@@ -944,7 +945,8 @@ void nodesDestroyNode(SNode* pNode) {
     case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
     case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
     case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
-    case QUERY_NODE_SHOW_TAGS_STMT: {
+    case QUERY_NODE_SHOW_TAGS_STMT:
+    case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: {
       SShowStmt* pStmt = (SShowStmt*)pNode;
       nodesDestroyNode(pStmt->pDbName);
       nodesDestroyNode(pStmt->pTbName);

source/libs/parser/inc/sql.y

@@ -101,6 +101,7 @@ cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C).
 %destructor privileges                                                            { }
 privileges(A) ::= ALL.                                                            { A = PRIVILEGE_TYPE_ALL; }
 privileges(A) ::= priv_type_list(B).                                              { A = B; }
+privileges(A) ::= SUBSCRIBE.                                                      { A = PRIVILEGE_TYPE_SUBSCRIBE; }
 
 %type priv_type_list                                                              { int64_t }
 %destructor priv_type_list                                                        { }
@@ -116,6 +117,7 @@ priv_type(A) ::= WRITE.
 %destructor priv_level                                                            { }
 priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR.                                      { A = B; }
 priv_level(A) ::= db_name(B) NK_DOT NK_STAR.                                      { A = B; }
+priv_level(A) ::= topic_name(B).                                                  { A = B; }
 
 /************************************************ create/drop/alter dnode *********************************************/
 cmd ::= CREATE DNODE dnode_endpoint(A).                                           { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
@@ -393,6 +395,7 @@ col_name(A) ::= column_name(B).
 /************************************************ show ****************************************************************/
 cmd ::= SHOW DNODES.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); }
 cmd ::= SHOW USERS.                                                               { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); }
+cmd ::= SHOW USER PRIVILEGES.                                                     { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); }
 cmd ::= SHOW DATABASES.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); }
 cmd ::= SHOW db_name_cond_opt(A) TABLES like_pattern_opt(B).                      { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, A, B, OP_TYPE_LIKE); }
 cmd ::= SHOW db_name_cond_opt(A) STABLES like_pattern_opt(B).                     { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, A, B, OP_TYPE_LIKE); }

source/libs/parser/src/parAstCreater.c

@@ -1815,7 +1815,7 @@ SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbN
   SGrantStmt* pStmt = (SGrantStmt*)nodesMakeNode(QUERY_NODE_GRANT_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   pStmt->privileges = privileges;
-  COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
+  COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName);
   COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
   return (SNode*)pStmt;
 }
@@ -1828,7 +1828,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb
   SRevokeStmt* pStmt = (SRevokeStmt*)nodesMakeNode(QUERY_NODE_REVOKE_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   pStmt->privileges = privileges;
-  COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
+  COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName);
   COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
   return (SNode*)pStmt;
 }

source/libs/parser/src/parAstParser.c

@@ -504,6 +504,11 @@ static int32_t collectMetaKeyFromShowVnodes(SCollectMetaKeyCxt* pCxt, SShowVnode
                                  pCxt->pMetaCache);
 }
 
+static int32_t collectMetaKeyFromShowUserPrivileges(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_PRIVILEGES,
+                                 pCxt->pMetaCache);
+}
+
 static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) {
   return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
 }
@@ -648,6 +653,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
       return collectMetaKeyFromShowDnodeVariables(pCxt, (SShowDnodeVariablesStmt*)pStmt);
     case QUERY_NODE_SHOW_VNODES_STMT:
       return collectMetaKeyFromShowVnodes(pCxt, (SShowVnodesStmt*)pStmt);
+    case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
+      return collectMetaKeyFromShowUserPrivileges(pCxt, (SShowStmt*)pStmt);
     case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
       return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt);
     case QUERY_NODE_SHOW_CREATE_TABLE_STMT:

source/libs/parser/src/parTokenizer.c

@@ -161,6 +161,7 @@ static SKeyword keywordTable[] = {
     {"PPS",                  TK_PPS},
     {"PRECISION",            TK_PRECISION},
     {"PREV",                 TK_PREV},
+    {"PRIVILEGES",           TK_PRIVILEGES},
     {"QNODE",                TK_QNODE},
     {"QNODES",               TK_QNODES},
     {"QTIME",                TK_QTIME},
@@ -202,6 +203,7 @@ static SKeyword keywordTable[] = {
     {"STREAM",               TK_STREAM},
     {"STREAMS",              TK_STREAMS},
     {"STRICT",               TK_STRICT},
+    {"SUBSCRIBE",            TK_SUBSCRIBE},
     {"SUBSCRIPTIONS",        TK_SUBSCRIPTIONS},
     {"SUBTABLE",             TK_SUBTABLE},
     {"SYSINFO",              TK_SYSINFO},

source/libs/parser/src/parTranslater.c

@@ -251,6 +251,12 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
     .numOfShowCols = 1,
     .pShowCols = {"*"}
   },
+  { .showType = QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
+    .pDbName = TSDB_INFORMATION_SCHEMA_DB,
+    .pTableName = TSDB_INS_TABLE_USER_PRIVILEGES,
+    .numOfShowCols = 1,
+    .pShowCols = {"*"}
+  },
 };
 // clang-format on
 
@@ -5031,7 +5037,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt
   alterReq.sysInfo = pStmt->sysinfo;
   snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password);
   if (NULL != pCxt->pParseCxt->db) {
-    snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db);
+    snprintf(alterReq.objname, sizeof(alterReq.objname), "%s", pCxt->pParseCxt->db);
   }
 
   return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq);
@@ -5710,9 +5716,11 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
     req.alterType = TSDB_ALTER_USER_ADD_READ_DB;
   } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
     req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB;
+  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
+    req.alterType = TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC;
   }
   strcpy(req.user, pStmt->userName);
-  sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName);
+  sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
   return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
 }
 
@@ -5726,9 +5734,11 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
     req.alterType = TSDB_ALTER_USER_REMOVE_READ_DB;
   } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
     req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB;
+  } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
+    req.alterType = TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC;
   }
   strcpy(req.user, pStmt->userName);
-  sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName);
+  sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
   return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
 }
 
@@ -7504,6 +7514,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
     case QUERY_NODE_SHOW_CONSUMERS_STMT:
     case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
     case QUERY_NODE_SHOW_TAGS_STMT:
+    case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
       code = rewriteShow(pCxt, pQuery);
       break;
     case QUERY_NODE_SHOW_VGROUPS_STMT:

source/libs/parser/test/mockCatalog.cpp

@@ -101,6 +101,10 @@ void generateInformationSchema(MockCatalogService* mcs) {
       .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN)
       .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN)
       .done();
+  mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_PRIVILEGES, TSDB_SYSTEM_TABLE, 2)
+      .addColumn("user_name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN)
+      .addColumn("privilege", TSDB_DATA_TYPE_BINARY, 10)
+      .done();
 }
 
 void generatePerformanceSchema(MockCatalogService* mcs) {
@@ -248,8 +252,8 @@ int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, con
   return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList);
 }
 
-int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId,
+int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum,
-                                int32_t* tableNum, int64_t* stateTs) {
+                                int64_t* stateTs) {
   return 0;
 }
 

source/libs/parser/test/parAlterToBalanceTest.cpp

@@ -613,7 +613,7 @@ TEST_F(ParserInitialATest, alterUser) {
     if (nullptr != pPass) {
       strcpy(expect.pass, pPass);
     }
-    strcpy(expect.dbname, "test");
+    strcpy(expect.objname, "test");
   };
 
   setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
@@ -627,7 +627,7 @@ TEST_F(ParserInitialATest, alterUser) {
     ASSERT_EQ(req.enable, expect.enable);
     ASSERT_EQ(std::string(req.user), std::string(expect.user));
     ASSERT_EQ(std::string(req.pass), std::string(expect.pass));
-    ASSERT_EQ(std::string(req.dbname), std::string(expect.dbname));
+    ASSERT_EQ(std::string(req.objname), std::string(expect.objname));
   });
 
   setAlterUserReq("wxy", TSDB_ALTER_USER_PASSWD, "123456");

source/libs/parser/test/parExplainToSyncdbTest.cpp

@@ -34,10 +34,38 @@ TEST_F(ParserExplainToSyncdbTest, explain) {
 TEST_F(ParserExplainToSyncdbTest, grant) {
   useDb("root", "test");
 
+  SAlterUserReq expect = {0};
+
+  auto setAlterUserReq = [&](int8_t alterType, const string& user, const string& obj) {
+    expect.alterType = alterType;
+    snprintf(expect.user, sizeof(expect.user), "%s", user.c_str());
+    snprintf(expect.objname, sizeof(expect.objname), "%s", obj.c_str());
+  };
+
+  setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
+    ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_GRANT_STMT);
+    ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_USER);
+    SAlterUserReq req = {0};
+    ASSERT_EQ(tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
+    ASSERT_EQ(req.alterType, expect.alterType);
+    ASSERT_EQ(string(req.user), string(expect.user));
+    ASSERT_EQ(string(req.objname), string(expect.objname));
+  });
+
+  setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.test");
   run("GRANT ALL ON test.* TO wxy");
+
+  setAlterUserReq(TSDB_ALTER_USER_ADD_READ_DB, "wxy", "0.test");
   run("GRANT READ ON test.* TO wxy");
+
+  setAlterUserReq(TSDB_ALTER_USER_ADD_WRITE_DB, "wxy", "0.test");
   run("GRANT WRITE ON test.* TO wxy");
+
+  setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.test");
   run("GRANT READ, WRITE ON test.* TO wxy");
+
+  setAlterUserReq(TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC, "wxy", "0.tp1");
+  run("GRANT SUBSCRIBE ON tp1 TO wxy");
 }
 
 TEST_F(ParserExplainToSyncdbTest, insert) {

source/libs/parser/test/parShowToUse.cpp

@@ -213,7 +213,13 @@ TEST_F(ParserShowToUseTest, showTags) {
 TEST_F(ParserShowToUseTest, showUsers) {
   useDb("root", "test");
 
-  run("SHOW users");
+  run("SHOW USERS");
+}
+
+TEST_F(ParserShowToUseTest, showUserPrivileges) {
+  useDb("root", "test");
+
+  run("SHOW USER PRIVILEGES");
 }
 
 TEST_F(ParserShowToUseTest, showVariables) {

source/libs/stream/src/streamUpdate.c

@@ -44,6 +44,11 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
   }
 }
 
+static void clearItemHelper(void* p) {
+  SScalableBf** pBf = p;
+  tScalableBfDestroy(*pBf);
+}
+
 static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) {
   if (count < pInfo->numSBFs) {
     for (uint64_t i = 0; i < count; ++i) {
@@ -52,7 +57,7 @@ static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) {
       taosArrayRemove(pInfo->pTsSBFs, 0);
     }
   } else {
-    taosArrayClearP(pInfo->pTsSBFs, (FDelete)tScalableBfDestroy);
+    taosArrayClearEx(pInfo->pTsSBFs, clearItemHelper);
   }
   pInfo->minTS += pInfo->interval * count;
 }

source/libs/sync/inc/syncCommit.h

@@ -47,8 +47,13 @@ extern "C" {
 //
 void syncOneReplicaAdvance(SSyncNode* pSyncNode);
 void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode);
+
 bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index);
 bool syncAgree(SSyncNode* pSyncNode, SyncIndex index);
+bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index);
+
+int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex);
+int64_t syncNodeCheckCommitIndex(SSyncNode* ths, SyncIndex indexLikely);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncInt.h

@@ -50,6 +50,8 @@ typedef struct SyncPreSnapshotReply   SyncPreSnapshotReply;
 typedef struct SyncHeartbeatReply     SyncHeartbeatReply;
 typedef struct SyncHeartbeat          SyncHeartbeat;
 typedef struct SyncPreSnapshot        SyncPreSnapshot;
+typedef struct SSyncLogBuffer         SSyncLogBuffer;
+typedef struct SSyncLogReplMgr        SSyncLogReplMgr;
 
 typedef struct SRaftId {
   SyncNodeId  addr;
@@ -97,6 +99,7 @@ typedef struct SSyncNode {
   char        configPath[TSDB_FILENAME_LEN * 2];
 
   // sync io
+  SSyncLogBuffer* pLogBuf;
   SWal*         pWal;
   const SMsgCb* msgcb;
   int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg);
@@ -179,6 +182,9 @@ typedef struct SSyncNode {
   SSyncSnapshotSender*   senders[TSDB_MAX_REPLICA];
   SSyncSnapshotReceiver* pNewNodeReceiver;
 
+  // log replication mgr
+  SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA];
+
   SPeerState peerStates[TSDB_MAX_REPLICA];
 
   // is config changing
@@ -197,6 +203,7 @@ typedef struct SSyncNode {
   int32_t configChangeNum;
   int32_t hbSlowNum;
   int32_t hbrSlowNum;
+  int32_t tmrRoutineNum;
 
   bool isStart;
 
@@ -204,11 +211,12 @@ typedef struct SSyncNode {
 
 // open/close --------------
 SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo);
-void       syncNodeStart(SSyncNode* pSyncNode);
+int32_t    syncNodeStart(SSyncNode* pSyncNode);
-void       syncNodeStartStandBy(SSyncNode* pSyncNode);
+int32_t    syncNodeStartStandBy(SSyncNode* pSyncNode);
 void       syncNodeClose(SSyncNode* pSyncNode);
 void       syncNodePreClose(SSyncNode* pSyncNode);
 int32_t    syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak);
+int32_t    syncNodeRestore(SSyncNode* pSyncNode);
 void       syncHbTimerDataFree(SSyncHbTimerData* pData);
 
 // on message ---------------------
@@ -243,6 +251,7 @@ int32_t   syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHa
 bool      syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode);
 bool      syncNodeSnapshotSending(SSyncNode* pSyncNode);
 bool      syncNodeSnapshotRecving(SSyncNode* pSyncNode);
+bool      syncNodeIsReadyForRead(SSyncNode* pSyncNode);
 
 // raft state change --------------
 void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term);
@@ -259,6 +268,9 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode);
 void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId);
 void syncNodeVoteForSelf(SSyncNode* pSyncNode);
 
+// log replication
+SSyncLogReplMgr* syncNodeGetLogReplMgr(SSyncNode* pNode, SRaftId* pDestId);
+
 // snapshot --------------
 bool    syncNodeHasSnapshot(SSyncNode* pSyncNode);
 void    syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode);

source/libs/sync/inc/syncMessage.h

@@ -106,7 +106,7 @@ typedef struct SyncAppendEntriesReply {
   SRaftId  destId;
   // private data
   SyncTerm  term;
-  SyncTerm  privateTerm;
+  SyncTerm  lastMatchTerm;
   bool      success;
   SyncIndex matchIndex;
   SyncIndex lastSendIndex;
@@ -246,6 +246,8 @@ int32_t syncBuildRequestVote(SRpcMsg* pMsg, int32_t vgId);
 int32_t syncBuildRequestVoteReply(SRpcMsg* pMsg, int32_t vgId);
 int32_t syncBuildAppendEntries(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId);
 int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId);
+int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm,
+                                          SRpcMsg* pRpcMsg);
 int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId);
 int32_t syncBuildHeartbeatReply(SRpcMsg* pMsg, int32_t vgId);
 int32_t syncBuildPreSnapshot(SRpcMsg* pMsg, int32_t vgId);

source/libs/sync/inc/syncPipeline.h

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_PIPELINE_H
+#define _TD_LIBS_SYNC_PIPELINE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "syncInt.h"
+
+typedef struct SSyncReplInfo {
+  bool    barrier;
+  bool    acked;
+  int64_t timeMs;
+  int64_t term;
+} SSyncReplInfo;
+
+typedef struct SSyncLogReplMgr {
+  SSyncReplInfo states[TSDB_SYNC_LOG_BUFFER_SIZE];
+  int64_t       startIndex;
+  int64_t       matchIndex;
+  int64_t       endIndex;
+  int64_t       size;
+  bool          restored;
+  int64_t       peerStartTime;
+  int32_t       retryBackoff;
+  int32_t       peerId;
+} SSyncLogReplMgr;
+
+typedef struct SSyncLogBufEntry {
+  SSyncRaftEntry* pItem;
+  SyncIndex       prevLogIndex;
+  SyncTerm        prevLogTerm;
+} SSyncLogBufEntry;
+
+typedef struct SSyncLogBuffer {
+  SSyncLogBufEntry entries[TSDB_SYNC_LOG_BUFFER_SIZE];
+  int64_t          startIndex;
+  int64_t          commitIndex;
+  int64_t          matchIndex;
+  int64_t          endIndex;
+  int64_t          size;
+  TdThreadMutex    mutex;
+  TdThreadMutexAttr attr;
+} SSyncLogBuffer;
+
+// SSyncLogRepMgr
+SSyncLogReplMgr* syncLogReplMgrCreate();
+void             syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr);
+int32_t          syncLogReplMgrReset(SSyncLogReplMgr* pMgr);
+
+int32_t          syncNodeLogReplMgrInit(SSyncNode* pNode);
+void             syncNodeLogReplMgrDestroy(SSyncNode* pNode);
+
+// access
+static FORCE_INLINE int64_t syncLogGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) {
+  return (1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS;
+}
+
+static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) {
+  return TMIN(pMgr->retryBackoff + 1, SYNC_MAX_RETRY_BACKOFF);
+}
+
+SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index);
+
+int32_t  syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
+int32_t  syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm,
+                                     SRaftId* pDestId, bool* pBarrier);
+int32_t  syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
+int32_t  syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index);
+
+int32_t  syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
+int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
+int32_t  syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
+
+int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg);
+int32_t  syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
+
+// SSyncLogBuffer
+SSyncLogBuffer* syncLogBufferCreate();
+void            syncLogBufferDestroy(SSyncLogBuffer* pBuf);
+int32_t         syncLogBufferInit(SSyncLogBuffer* pBuf, SSyncNode* pNode);
+int32_t         syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode);
+
+// access
+int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf);
+int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry);
+int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm);
+int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm);
+int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t commitIndex);
+int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode);
+
+// private
+SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex index, bool* pInBuf);
+int32_t         syncLogBufferValidate(SSyncLogBuffer* pBuf);
+int32_t         syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_PIPELINE_H*/

source/libs/sync/inc/syncRaftEntry.h

@@ -42,9 +42,13 @@ SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, S
 SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index);
 SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg);
 SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId);
-void            syncEntryDestory(SSyncRaftEntry* pEntry);
+void            syncEntryDestroy(SSyncRaftEntry* pEntry);
 void            syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg);  // step 7
 
+static FORCE_INLINE bool syncLogIsReplicationBarrier(SSyncRaftEntry* pEntry) {
+  return pEntry->originalRpcType == TDMT_SYNC_NOOP;
+}
+
 typedef struct SRaftEntryHashCache {
   SHashObj*     pEntryHash;
   int32_t       maxCount;

source/libs/sync/inc/syncReplication.h

@@ -52,6 +52,10 @@ int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* pDestId, SRpc
 
 int32_t syncNodeReplicate(SSyncNode* pSyncNode);
 int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapshot);
+int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode);
+
+int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
+int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncSnapshot.h

@@ -87,6 +87,8 @@ void                   snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceive
 int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg);
 int32_t syncNodeOnSnapshotReply(SSyncNode *ths, const SRpcMsg *pMsg);
 
+SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode *pSyncNode, SyncIndex snapshotLastApplyIndex);
+
 // start
 
 #ifdef __cplusplus

source/libs/sync/inc/syncUtil.h

@@ -94,11 +94,12 @@ void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const c
 void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s);
 void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s);
 
-void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed, int64_t execTime);
+void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed,
-void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64_t timeDiff);
+                          int64_t execTime);
+void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64_t timeDiff, const char* s);
 
 void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s);
-void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff);
+void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff, const char* s);
 
 void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s);
 void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s);

source/libs/sync/src/syncAppendEntries.c

@@ -15,10 +15,13 @@
 
 #define _DEFAULT_SOURCE
 #include "syncAppendEntries.h"
+#include "syncPipeline.h"
 #include "syncMessage.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
+#include "syncReplication.h"
 #include "syncUtil.h"
+#include "syncCommit.h"
 
 // TLA+ Spec
 // HandleAppendEntriesRequest(i, j, m) ==
@@ -124,9 +127,113 @@ int32_t syncNodeFollowerCommit(SSyncNode* ths, SyncIndex newCommitIndex) {
   return 0;
 }
 
+SSyncRaftEntry* syncLogAppendEntriesToRaftEntry(const SyncAppendEntries* pMsg) {
+  SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen);
+  if (pEntry == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  (void)memcpy(pEntry, pMsg->data, pMsg->dataLen);
+  ASSERT(pEntry->bytes == pMsg->dataLen);
+  return pEntry;
+}
+
 int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   SyncAppendEntries* pMsg = pRpcMsg->pCont;
   SRpcMsg            rpcRsp = {0};
+  bool               accepted = false;
+  // if already drop replica, do not process
+  if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
+    syncLogRecvAppendEntries(ths, pMsg, "not in my config");
+    goto _IGNORE;
+  }
+
+  int32_t code = syncBuildAppendEntriesReply(&rpcRsp, ths->vgId);
+  if (code != 0) {
+    syncLogRecvAppendEntries(ths, pMsg, "build rsp error");
+    goto _IGNORE;
+  }
+
+  SyncAppendEntriesReply* pReply = rpcRsp.pCont;
+  // prepare response msg
+  pReply->srcId = ths->myRaftId;
+  pReply->destId = pMsg->srcId;
+  pReply->term = ths->pRaftStore->currentTerm;
+  pReply->success = false;
+  pReply->matchIndex = SYNC_INDEX_INVALID;
+  pReply->lastSendIndex = pMsg->prevLogIndex + 1;
+  pReply->startTime = ths->startTime;
+
+  if (pMsg->term < ths->pRaftStore->currentTerm) {
+    goto _SEND_RESPONSE;
+  }
+
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    pReply->term = pMsg->term;
+  }
+
+  syncNodeStepDown(ths, pMsg->term);
+  syncNodeResetElectTimer(ths);
+
+  if (pMsg->dataLen < (int32_t)sizeof(SSyncRaftEntry)) {
+    sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d",
+           ths->vgId, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
+    goto _IGNORE;
+  }
+
+  SSyncRaftEntry* pEntry = syncLogAppendEntriesToRaftEntry(pMsg);
+
+  if (pEntry == NULL) {
+    sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr());
+    goto _IGNORE;
+  }
+
+  if (pMsg->prevLogIndex + 1 != pEntry->index || pEntry->term < 0) {
+    sError("vgId:%d, invalid previous log index in msg. index:%" PRId64 ",  term:%" PRId64 ", prevLogIndex:%" PRId64
+           ", prevLogTerm:%" PRId64,
+           ths->vgId, pEntry->index, pEntry->term, pMsg->prevLogIndex, pMsg->prevLogTerm);
+    goto _IGNORE;
+  }
+
+  sTrace("vgId:%d, recv append entries msg. index:%" PRId64 ", term:%" PRId64 ", preLogIndex:%" PRId64
+         ", prevLogTerm:%" PRId64 " commitIndex:%" PRId64 "",
+         pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex);
+
+  // accept
+  if (syncLogBufferAccept(ths->pLogBuf, ths, pEntry, pMsg->prevLogTerm) < 0) {
+    goto _SEND_RESPONSE;
+  }
+  accepted = true;
+
+_SEND_RESPONSE:
+  pReply->matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, &pReply->lastMatchTerm);
+  bool matched = (pReply->matchIndex >= pReply->lastSendIndex);
+  if (accepted && matched) {
+    pReply->success = true;
+    // update commit index only after matching
+    (void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex);
+  }
+
+  // ack, i.e. send response
+  (void)syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp);
+
+  // commit index, i.e. leader notice me
+  if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) {
+    sError("vgId:%d, failed to commit raft fsm log since %s.", ths->vgId, terrstr());
+    goto _out;
+  }
+
+_out:
+  return 0;
+
+_IGNORE:
+  rpcFreeCont(rpcRsp.pCont);
+  return 0;
+}
+
+int32_t syncNodeOnAppendEntriesOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
+  SyncAppendEntries* pMsg = pRpcMsg->pCont;
+  SRpcMsg            rpcRsp = {0};
 
   // if already drop replica, do not process
   if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
@@ -233,18 +340,20 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
           if (hLocal) {
             taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
           } else {
-            syncEntryDestory(pLocalEntry);
+            syncEntryDestroy(pLocalEntry);
           }
 
           if (hAppend) {
             taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
           } else {
-            syncEntryDestory(pAppendEntry);
+            syncEntryDestroy(pAppendEntry);
           }
 
           goto _IGNORE;
         }
 
+        ASSERT(pAppendEntry->index == appendIndex);
+
         // append
         code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
         if (code != 0) {
@@ -255,13 +364,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
           if (hLocal) {
             taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
           } else {
-            syncEntryDestory(pLocalEntry);
+            syncEntryDestroy(pLocalEntry);
           }
 
           if (hAppend) {
             taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
           } else {
-            syncEntryDestory(pAppendEntry);
+            syncEntryDestroy(pAppendEntry);
           }
 
           goto _IGNORE;
@@ -281,8 +390,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
           snprintf(logBuf, sizeof(logBuf), "ignore, log not exist, truncate error, append-index:%" PRId64, appendIndex);
           syncLogRecvAppendEntries(ths, pMsg, logBuf);
 
-          syncEntryDestory(pLocalEntry);
+          syncEntryDestroy(pLocalEntry);
-          syncEntryDestory(pAppendEntry);
+          syncEntryDestroy(pAppendEntry);
           goto _IGNORE;
         }
 
@@ -296,13 +405,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
           if (hLocal) {
             taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
           } else {
-            syncEntryDestory(pLocalEntry);
+            syncEntryDestroy(pLocalEntry);
           }
 
           if (hAppend) {
             taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
           } else {
-            syncEntryDestory(pAppendEntry);
+            syncEntryDestroy(pAppendEntry);
           }
 
           goto _IGNORE;
@@ -320,13 +429,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
         if (hLocal) {
           taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
         } else {
-          syncEntryDestory(pLocalEntry);
+          syncEntryDestroy(pLocalEntry);
         }
 
         if (hAppend) {
           taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
         } else {
-          syncEntryDestory(pAppendEntry);
+          syncEntryDestroy(pAppendEntry);
         }
 
         goto _IGNORE;
@@ -339,13 +448,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
     if (hLocal) {
       taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
     } else {
-      syncEntryDestory(pLocalEntry);
+      syncEntryDestroy(pLocalEntry);
     }
 
     if (hAppend) {
       taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
     } else {
-      syncEntryDestory(pAppendEntry);
+      syncEntryDestroy(pAppendEntry);
     }
 
   } else {

source/libs/sync/src/syncAppendEntriesReply.c

@@ -17,7 +17,9 @@
 #include "syncAppendEntriesReply.h"
 #include "syncCommit.h"
 #include "syncIndexMgr.h"
+#include "syncPipeline.h"
 #include "syncMessage.h"
+#include "syncRaftEntry.h"
 #include "syncRaftStore.h"
 #include "syncReplication.h"
 #include "syncSnapshot.h"
@@ -38,8 +40,58 @@
 //
 
 int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
-  int32_t                 ret = 0;
   SyncAppendEntriesReply* pMsg = pRpcMsg->pCont;
+  int32_t ret = 0;
+
+  // if already drop replica, do not process
+  if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
+    syncLogRecvAppendEntriesReply(ths, pMsg, "not in my config");
+    return 0;
+  }
+
+  // drop stale response
+  if (pMsg->term < ths->pRaftStore->currentTerm) {
+    syncLogRecvAppendEntriesReply(ths, pMsg, "drop stale response");
+    return 0;
+  }
+
+  if (ths->state == TAOS_SYNC_STATE_LEADER) {
+    if (pMsg->term > ths->pRaftStore->currentTerm) {
+      syncLogRecvAppendEntriesReply(ths, pMsg, "error term");
+      syncNodeStepDown(ths, pMsg->term);
+      return -1;
+    }
+
+    ASSERT(pMsg->term == ths->pRaftStore->currentTerm);
+
+    sTrace("vgId:%d received append entries reply. srcId:0x%016" PRIx64 ",  term:%" PRId64 ", matchIndex:%" PRId64 "",
+           pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex);
+
+    if (pMsg->success) {
+      SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId));
+      if (pMsg->matchIndex > oldMatchIndex) {
+        syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex);
+      }
+
+      // commit if needed
+      SyncIndex indexLikely = TMIN(pMsg->matchIndex, ths->pLogBuf->matchIndex);
+      SyncIndex commitIndex = syncNodeCheckCommitIndex(ths, indexLikely);
+      (void)syncLogBufferCommit(ths->pLogBuf, ths, commitIndex);
+    }
+
+    // replicate log
+    SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(ths, &pMsg->srcId);
+    if (pMgr == NULL) {
+      sError("vgId:%d, failed to get log repl mgr for src addr: 0x%016" PRIx64, ths->vgId, pMsg->srcId.addr);
+      return -1;
+    }
+    (void)syncLogReplMgrProcessReply(pMgr, ths, pMsg);
+  }
+  return 0;
+}
+
+int32_t syncNodeOnAppendEntriesReplyOld(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
+  int32_t ret = 0;
 
   // if already drop replica, do not process
   if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {

source/libs/sync/src/syncCommit.c

@@ -84,6 +84,7 @@ void syncOneReplicaAdvance(SSyncNode* pSyncNode) {
 }
 
 void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
+  ASSERT(false && "deprecated");
   if (pSyncNode == NULL) {
     sError("pSyncNode is NULL");
     return;
@@ -138,7 +139,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
         if (h) {
           taosLRUCacheRelease(pCache, h, false);
         } else {
-          syncEntryDestory(pEntry);
+          syncEntryDestroy(pEntry);
         }
 
         break;
@@ -150,7 +151,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
       if (h) {
         taosLRUCacheRelease(pCache, h, false);
       } else {
-        syncEntryDestory(pEntry);
+        syncEntryDestroy(pEntry);
       }
     }
   }
@@ -286,15 +287,48 @@ bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) {
 }
 */
 
-bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) {
+bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) {
+  int            count = 0;
+  SSyncIndexMgr* pMatches = pNode->pMatchIndex;
+  ASSERT(pNode->replicaNum == pMatches->replicaNum);
+
+  for (int i = 0; i < pNode->replicaNum; i++) {
+    SyncIndex matchIndex = pMatches->index[i];
+    if (matchIndex >= index) {
+      count++;
+    }
+  }
+
+  return count >= pNode->quorum;
+}
+
+bool syncAgree(SSyncNode* pNode, SyncIndex index) {
   int agreeCount = 0;
-  for (int i = 0; i < pSyncNode->replicaNum; ++i) {
+  for (int i = 0; i < pNode->replicaNum; ++i) {
-    if (syncAgreeIndex(pSyncNode, &(pSyncNode->replicasId[i]), index)) {
+    if (syncAgreeIndex(pNode, &(pNode->replicasId[i]), index)) {
       ++agreeCount;
     }
-    if (agreeCount >= pSyncNode->quorum) {
+    if (agreeCount >= pNode->quorum) {
       return true;
     }
   }
   return false;
 }
+
+int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex) {
+  SyncIndex lastVer = ths->pLogStore->syncLogLastIndex(ths->pLogStore);
+  commitIndex = TMAX(commitIndex, ths->commitIndex);
+  ths->commitIndex = TMIN(commitIndex, lastVer);
+  ths->pLogStore->syncLogUpdateCommitIndex(ths->pLogStore, ths->commitIndex);
+  return ths->commitIndex;
+}
+
+int64_t syncNodeCheckCommitIndex(SSyncNode* ths, SyncIndex indexLikely) {
+  if (indexLikely > ths->commitIndex && syncNodeAgreedUpon(ths, indexLikely)) {
+    SyncIndex commitIndex = indexLikely;
+    syncNodeUpdateCommitIndex(ths, commitIndex);
+    sTrace("vgId:%d, agreed upon. role:%d, term:%" PRId64 ", index: %" PRId64 "", ths->vgId, ths->state,
+           ths->pRaftStore->currentTerm, commitIndex);
+  }
+  return ths->commitIndex;
+}

source/libs/sync/src/syncEnv.c

@@ -100,7 +100,9 @@ SSyncNode *syncNodeAcquire(int64_t rid) {
   return pNode;
 }
 
-void syncNodeRelease(SSyncNode *pNode) { taosReleaseRef(gNodeRefId, pNode->rid); }
+void syncNodeRelease(SSyncNode *pNode) {
+  if (pNode) taosReleaseRef(gNodeRefId, pNode->rid);
+}
 
 int64_t syncHbTimerDataAdd(SSyncHbTimerData *pData) {
   pData->rid = taosAddRef(gHbDataRefId, pData);

source/libs/sync/src/syncIndexMgr.c

@@ -81,6 +81,15 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId,
          index);
 }
 
+SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pDestId) {
+  for (int i = 0; i < pNode->replicaNum; i++) {
+    if (syncUtilSameId(&(pNode->replicasId[i]), pDestId)) {
+      return pNode->logReplMgrs[i];
+    }
+  }
+  return NULL;
+}
+
 SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
   if (pSyncIndexMgr == NULL) {
     return SYNC_INDEX_INVALID;

source/libs/sync/src/syncMain.c

@@ -22,6 +22,7 @@
 #include "syncEnv.h"
 #include "syncIndexMgr.h"
 #include "syncInt.h"
+#include "syncPipeline.h"
 #include "syncMessage.h"
 #include "syncRaftCfg.h"
 #include "syncRaftLog.h"
@@ -34,6 +35,7 @@
 #include "syncTimeout.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
+#include "tref.h"
 
 static void    syncNodeEqPingTimer(void* param, void* tmrId);
 static void    syncNodeEqElectTimer(void* param, void* tmrId);
@@ -56,7 +58,6 @@ static int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeade
 static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry);
 
 static ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode);
-static SyncIndex     syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapshotLastApplyIndex);
 
 int64_t syncOpen(SSyncInfo* pSyncInfo) {
   SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
@@ -80,12 +81,29 @@ int64_t syncOpen(SSyncInfo* pSyncInfo) {
   return pSyncNode->rid;
 }
 
-void syncStart(int64_t rid) {
+int32_t syncStart(int64_t rid) {
   SSyncNode* pSyncNode = syncNodeAcquire(rid);
-  if (pSyncNode != NULL) {
+  if (pSyncNode == NULL) {
-    syncNodeStart(pSyncNode);
+    sError("failed to acquire rid: %" PRId64 " of tsNodeReftId for pSyncNode", rid);
-    syncNodeRelease(pSyncNode);
+    return -1;
   }
+
+  if (syncNodeRestore(pSyncNode) < 0) {
+    sError("vgId:%d, failed to restore sync log buffer since %s", pSyncNode->vgId, terrstr());
+    goto _err;
+  }
+
+  if (syncNodeStart(pSyncNode) < 0) {
+    sError("vgId:%d, failed to start sync node since %s", pSyncNode->vgId, terrstr());
+    goto _err;
+  }
+
+  syncNodeRelease(pSyncNode);
+  return 0;
+
+_err:
+  syncNodeRelease(pSyncNode);
+  return -1;
 }
 
 void syncStop(int64_t rid) {
@@ -381,15 +399,13 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm) {
   return 0;
 }
 
-bool syncIsReadyForRead(int64_t rid) {
+bool syncNodeIsReadyForRead(SSyncNode* pSyncNode) {
-  SSyncNode* pSyncNode = syncNodeAcquire(rid);
   if (pSyncNode == NULL) {
     sError("sync ready for read error");
     return false;
   }
 
   if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) {
-    syncNodeRelease(pSyncNode);
     return true;
   }
 
@@ -428,7 +444,7 @@ bool syncIsReadyForRead(int64_t rid) {
           if (h) {
             taosLRUCacheRelease(pCache, h, false);
           } else {
-            syncEntryDestory(pEntry);
+            syncEntryDestroy(pEntry);
           }
         }
       }
@@ -443,6 +459,18 @@ bool syncIsReadyForRead(int64_t rid) {
     }
   }
 
+  return ready;
+}
+
+bool syncIsReadyForRead(int64_t rid) {
+  SSyncNode* pSyncNode = syncNodeAcquire(rid);
+  if (pSyncNode == NULL) {
+    sError("sync ready for read error");
+    return false;
+  }
+
+  bool ready = syncNodeIsReadyForRead(pSyncNode);
+
   syncNodeRelease(pSyncNode);
   return ready;
 }
@@ -521,6 +549,7 @@ SSyncState syncGetState(int64_t rid) {
   if (pSyncNode != NULL) {
     state.state = pSyncNode->state;
     state.restored = pSyncNode->restoreFinish;
+    state.canRead = syncNodeIsReadyForRead(pSyncNode);
     syncNodeRelease(pSyncNode);
   }
 
@@ -543,7 +572,7 @@ int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapsho
   int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index, &pEntry);
   if (code != 0) {
     if (pEntry != NULL) {
-      syncEntryDestory(pEntry);
+      syncEntryDestroy(pEntry);
     }
     syncNodeRelease(pSyncNode);
     return -1;
@@ -555,7 +584,7 @@ int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapsho
   pSnapshot->lastApplyTerm = pEntry->term;
   pSnapshot->lastConfigIndex = syncNodeGetSnapshotConfigIndex(pSyncNode, index);
 
-  syncEntryDestory(pEntry);
+  syncEntryDestroy(pEntry);
   syncNodeRelease(pSyncNode);
   return 0;
 }
@@ -626,7 +655,7 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) {
     tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN);
     pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort;
     pEpSet->numOfEps++;
-    sInfo("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port);
+    sDebug("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port);
   }
   if (pEpSet->numOfEps > 0) {
     pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps;
@@ -756,6 +785,29 @@ static int32_t syncHbTimerStop(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) {
   return ret;
 }
 
+int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) {
+  ASSERT(pNode->pLogStore != NULL && "log store not created");
+  ASSERT(pNode->pFsm != NULL && "pFsm not registered");
+  ASSERT(pNode->pFsm->FpGetSnapshotInfo != NULL && "FpGetSnapshotInfo not registered");
+  SSnapshot snapshot;
+  if (pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot) < 0) {
+    sError("vgId:%d, failed to get snapshot info since %s", pNode->vgId, terrstr());
+    return -1;
+  }
+  SyncIndex commitIndex = snapshot.lastApplyIndex;
+  SyncIndex firstVer = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore);
+  SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore);
+  if (lastVer < commitIndex || firstVer > commitIndex + 1) {
+    if (pNode->pLogStore->syncLogRestoreFromSnapshot(pNode->pLogStore, commitIndex)) {
+      sError("vgId:%d, failed to restore log store from snapshot since %s. lastVer: %" PRId64 ", snapshotVer: %" PRId64,
+             pNode->vgId, terrstr(), lastVer, commitIndex);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+// open/close --------------
 SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
   SSyncNode* pSyncNode = taosMemoryCalloc(1, sizeof(SSyncNode));
   if (pSyncNode == NULL) {
@@ -831,6 +883,13 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
   pSyncNode->syncEqMsg = pSyncInfo->syncEqMsg;
   pSyncNode->syncEqCtrlMsg = pSyncInfo->syncEqCtrlMsg;
 
+  // create raft log ring buffer
+  pSyncNode->pLogBuf = syncLogBufferCreate();
+  if (pSyncNode->pLogBuf == NULL) {
+    sError("failed to init sync log buffer since %s. vgId:%d", terrstr(), pSyncNode->vgId);
+    goto _error;
+  }
+
   // init raft config
   pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
   if (pSyncNode->pRaftCfg == NULL) {
@@ -956,6 +1015,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
   }
   pSyncNode->commitIndex = commitIndex;
 
+  if (syncNodeLogStoreRestoreOnNeed(pSyncNode) < 0) {
+    goto _error;
+  }
   // timer ms init
   pSyncNode->pingBaseLine = PING_TIMER_MS;
   pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN;
@@ -1013,9 +1075,13 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
   // is config changing
   pSyncNode->changing = false;
 
+  // replication mgr
+  syncNodeLogReplMgrInit(pSyncNode);
+
   // peer state
   syncNodePeerStateInit(pSyncNode);
 
+  //
   // min match index
   pSyncNode->minMatchIndex = SYNC_INDEX_INVALID;
 
@@ -1031,12 +1097,19 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
   // snapshotting
   atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID);
 
+  // init log buffer
+  if (syncLogBufferInit(pSyncNode->pLogBuf, pSyncNode) < 0) {
+    sError("vgId:%d, failed to init sync log buffer since %s", pSyncNode->vgId, terrstr());
+    goto _error;
+  }
+
   pSyncNode->isStart = true;
   pSyncNode->electNum = 0;
   pSyncNode->becomeLeaderNum = 0;
   pSyncNode->configChangeNum = 0;
   pSyncNode->hbSlowNum = 0;
   pSyncNode->hbrSlowNum = 0;
+  pSyncNode->tmrRoutineNum = 0;
 
   sNTrace(pSyncNode, "sync open, node:%p", pSyncNode);
 
@@ -1063,7 +1136,49 @@ void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) {
   }
 }
 
-void syncNodeStart(SSyncNode* pSyncNode) {
+int32_t syncNodeRestore(SSyncNode* pSyncNode) {
+  ASSERT(pSyncNode->pLogStore != NULL && "log store not created");
+  ASSERT(pSyncNode->pLogBuf != NULL && "ring log buffer not created");
+
+  SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+  SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore);
+  SyncIndex endIndex = pSyncNode->pLogBuf->endIndex;
+  if (lastVer != -1 && endIndex != lastVer + 1) {
+    terrno = TSDB_CODE_WAL_LOG_INCOMPLETE;
+    sError("vgId:%d, failed to restore sync node since %s. expected lastLogIndex: %" PRId64 ", lastVer: %" PRId64 "",
+           pSyncNode->vgId, terrstr(), endIndex - 1, lastVer);
+    return -1;
+  }
+
+  ASSERT(endIndex == lastVer + 1);
+  commitIndex = TMAX(pSyncNode->commitIndex, commitIndex);
+
+  if (syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, commitIndex) < 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t syncNodeStart(SSyncNode* pSyncNode) {
+  // start raft
+  if (pSyncNode->replicaNum == 1) {
+    raftStoreNextTerm(pSyncNode->pRaftStore);
+    syncNodeBecomeLeader(pSyncNode, "one replica start");
+
+    // Raft 3.6.2 Committing entries from previous terms
+    syncNodeAppendNoop(pSyncNode);
+  } else {
+    syncNodeBecomeFollower(pSyncNode, "first start");
+  }
+
+  int32_t ret = 0;
+  ret = syncNodeStartPingTimer(pSyncNode);
+  ASSERT(ret == 0);
+  return ret;
+}
+
+void syncNodeStartOld(SSyncNode* pSyncNode) {
   // start raft
   if (pSyncNode->replicaNum == 1) {
     raftStoreNextTerm(pSyncNode->pRaftStore);
@@ -1082,7 +1197,7 @@ void syncNodeStart(SSyncNode* pSyncNode) {
   ASSERT(ret == 0);
 }
 
-void syncNodeStartStandBy(SSyncNode* pSyncNode) {
+int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) {
   // state change
   pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
   syncNodeStopHeartbeatTimer(pSyncNode);
@@ -1095,6 +1210,7 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) {
   ret = 0;
   ret = syncNodeStartPingTimer(pSyncNode);
   ASSERT(ret == 0);
+  return ret;
 }
 
 void syncNodePreClose(SSyncNode* pSyncNode) {
@@ -1115,6 +1231,7 @@ void syncNodeClose(SSyncNode* pSyncNode) {
   ASSERT(ret == 0);
   pSyncNode->pRaftStore = NULL;
 
+  syncNodeLogReplMgrDestroy(pSyncNode);
   syncRespMgrDestroy(pSyncNode->pSyncRespMgr);
   pSyncNode->pSyncRespMgr = NULL;
   voteGrantedDestroy(pSyncNode->pVotesGranted);
@@ -1127,6 +1244,8 @@ void syncNodeClose(SSyncNode* pSyncNode) {
   pSyncNode->pMatchIndex = NULL;
   logStoreDestory(pSyncNode->pLogStore);
   pSyncNode->pLogStore = NULL;
+  syncLogBufferDestroy(pSyncNode->pLogBuf);
+  pSyncNode->pLogBuf = NULL;
   raftCfgClose(pSyncNode->pRaftCfg);
   pSyncNode->pRaftCfg = NULL;
 
@@ -1141,12 +1260,21 @@ void syncNodeClose(SSyncNode* pSyncNode) {
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
     if ((pSyncNode->senders)[i] != NULL) {
       sSTrace((pSyncNode->senders)[i], "snapshot sender destroy while close, data:%p", (pSyncNode->senders)[i]);
+
+      if (snapshotSenderIsStart((pSyncNode->senders)[i])) {
+        snapshotSenderStop((pSyncNode->senders)[i], false);
+      }
+
       snapshotSenderDestroy((pSyncNode->senders)[i]);
       (pSyncNode->senders)[i] = NULL;
     }
   }
 
   if (pSyncNode->pNewNodeReceiver != NULL) {
+    if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) {
+      snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver);
+    }
+
     snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver);
     pSyncNode->pNewNodeReceiver = NULL;
   }
@@ -1519,7 +1647,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
 
       // Raft 3.6.2 Committing entries from previous terms
       syncNodeAppendNoop(pSyncNode);
-      syncMaybeAdvanceCommitIndex(pSyncNode);
+      // syncMaybeAdvanceCommitIndex(pSyncNode);
 
     } else {
       syncNodeBecomeFollower(pSyncNode, tmpbuf);
@@ -1607,6 +1735,9 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) {
   // min match index
   pSyncNode->minMatchIndex = SYNC_INDEX_INVALID;
 
+  // reset log buffer
+  syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode);
+
   // trace log
   sNTrace(pSyncNode, "become follower %s", debugStr);
 }
@@ -1702,6 +1833,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) {
   // min match index
   pSyncNode->minMatchIndex = SYNC_INDEX_INVALID;
 
+  // reset log buffer
+  syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode);
+
   // trace log
   sNInfo(pSyncNode, "become leader %s", debugStr);
 }
@@ -1713,6 +1847,22 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
 
   sNTrace(pSyncNode, "state change syncNodeCandidate2Leader");
 
+  int32_t ret = syncNodeAppendNoop(pSyncNode);
+  if (ret < 0) {
+    sError("vgId:%d, failed to append noop entry since %s", pSyncNode->vgId, terrstr());
+  }
+
+  SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+  ASSERT(lastIndex >= 0);
+  sInfo("vgId:%d, become leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64 "",
+        pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
+}
+
+void syncNodeCandidate2LeaderOld(SSyncNode* pSyncNode) {
+  ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
+  ASSERT(voteGrantedMajority(pSyncNode->pVotesGranted));
+  syncNodeBecomeLeader(pSyncNode, "candidate to leader");
+
   // Raft 3.6.2 Committing entries from previous terms
   syncNodeAppendNoop(pSyncNode);
   syncMaybeAdvanceCommitIndex(pSyncNode);
@@ -1736,18 +1886,30 @@ int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) {
 void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
   ASSERT(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER);
   pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE;
+  SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+  sInfo("vgId:%d, become candidate from follower. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64,
+        pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
+
   sNTrace(pSyncNode, "follower to candidate");
 }
 
 void syncNodeLeader2Follower(SSyncNode* pSyncNode) {
   ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER);
   syncNodeBecomeFollower(pSyncNode, "leader to follower");
+  SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+  sInfo("vgId:%d, become follower from leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64,
+        pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
+
   sNTrace(pSyncNode, "leader to follower");
 }
 
 void syncNodeCandidate2Follower(SSyncNode* pSyncNode) {
   ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
   syncNodeBecomeFollower(pSyncNode, "candidate to follower");
+  SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+  sInfo("vgId:%d, become follower from candidate. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64,
+        pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
+
   sNTrace(pSyncNode, "candidate to follower");
 }
 
@@ -1901,7 +2063,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
     if (h) {
       taosLRUCacheRelease(pCache, h, false);
     } else {
-      syncEntryDestory(pPreEntry);
+      syncEntryDestroy(pPreEntry);
     }
 
     return preTerm;
@@ -2143,7 +2305,7 @@ static int32_t syncNodeEqNoop(SSyncNode* pNode) {
 
   SRpcMsg rpcMsg = {0};
   int32_t code = syncBuildClientRequestFromNoopEntry(&rpcMsg, pEntry, pNode->vgId);
-  syncEntryDestory(pEntry);
+  syncEntryDestroy(pEntry);
 
   sNTrace(pNode, "propose msg, type:noop");
   code = (*pNode->syncEqMsg)(pNode->msgcb, &rpcMsg);
@@ -2171,6 +2333,37 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand
   return code;
 }
 
+int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) {
+  // append to log buffer
+  if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) {
+    sError("vgId:%d, failed to enqueue sync log buffer. index:%" PRId64 "", ths->vgId, pEntry->index);
+    return -1;
+  }
+
+  // proceed match index, with replicating on needed
+  SyncIndex matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, NULL);
+
+  sTrace("vgId:%d, append raft entry. index: %" PRId64 ", term: %" PRId64 " pBuf: [%" PRId64 " %" PRId64 " %" PRId64
+         ", %" PRId64 ")",
+         ths->vgId, pEntry->index, pEntry->term, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex,
+         ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex);
+
+  // multi replica
+  if (ths->replicaNum > 1) {
+    return 0;
+  }
+
+  // single replica
+  (void)syncNodeUpdateCommitIndex(ths, matchIndex);
+
+  if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) {
+    sError("vgId:%d, failed to commit until commitIndex:%" PRId64 "", ths->vgId, ths->commitIndex);
+    return -1;
+  }
+
+  return 0;
+}
+
 bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) {
   if (pSyncNode->replicaNum == 1) {
     return false;
@@ -2214,6 +2407,20 @@ bool syncNodeSnapshotRecving(SSyncNode* pSyncNode) {
 }
 
 static int32_t syncNodeAppendNoop(SSyncNode* ths) {
+  SyncIndex index = syncLogBufferGetEndIndex(ths->pLogBuf);
+  SyncTerm  term = ths->pRaftStore->currentTerm;
+
+  SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId);
+  if (pEntry == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  int32_t ret = syncNodeAppend(ths, pEntry);
+  return 0;
+}
+
+static int32_t syncNodeAppendNoopOld(SSyncNode* ths) {
   int32_t ret = 0;
 
   SyncIndex       index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore);
@@ -2236,7 +2443,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
   if (h) {
     taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
   } else {
-    syncEntryDestory(pEntry);
+    syncEntryDestroy(pEntry);
   }
 
   return ret;
@@ -2245,9 +2452,13 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
 int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   SyncHeartbeat* pMsg = pRpcMsg->pCont;
 
+  const STraceId* trace = &pRpcMsg->info.traceId;
+  char            tbuf[40] = {0};
+  TRACE_TO_STR(trace, tbuf);
+
   int64_t tsMs = taosGetTimestampMs();
   int64_t timeDiff = tsMs - pMsg->timeStamp;
-  syncLogRecvHeartbeat(ths, pMsg, timeDiff);
+  syncLogRecvHeartbeat(ths, pMsg, timeDiff, tbuf);
 
   SRpcMsg rpcMsg = {0};
   (void)syncBuildHeartbeatReply(&rpcMsg, ths->vgId);
@@ -2257,6 +2468,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   pMsgReply->srcId = ths->myRaftId;
   pMsgReply->term = ths->pRaftStore->currentTerm;
   pMsgReply->privateTerm = 8864;  // magic number
+  pMsgReply->startTime = ths->startTime;
   pMsgReply->timeStamp = tsMs;
 
   if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) {
@@ -2320,11 +2532,35 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
 }
 
 int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
+  const STraceId* trace = &pRpcMsg->info.traceId;
+  char            tbuf[40] = {0};
+  TRACE_TO_STR(trace, tbuf);
+
   SyncHeartbeatReply* pMsg = pRpcMsg->pCont;
+  SSyncLogReplMgr*    pMgr = syncNodeGetLogReplMgr(ths, &pMsg->srcId);
+  if (pMgr == NULL) {
+    sError("vgId:%d, failed to get log repl mgr for the peer at addr 0x016%" PRIx64 "", ths->vgId, pMsg->srcId.addr);
+    return -1;
+  }
+
+  int64_t tsMs = taosGetTimestampMs();
+  syncLogRecvHeartbeatReply(ths, pMsg, tsMs - pMsg->timeStamp, tbuf);
+
+  syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs);
+
+  return syncLogReplMgrProcessHeartbeatReply(pMgr, ths, pMsg);
+}
+
+int32_t syncNodeOnHeartbeatReplyOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
+  SyncHeartbeatReply* pMsg = pRpcMsg->pCont;
+
+  const STraceId* trace = &pRpcMsg->info.traceId;
+  char            tbuf[40] = {0};
+  TRACE_TO_STR(trace, tbuf);
 
   int64_t tsMs = taosGetTimestampMs();
   int64_t timeDiff = tsMs - pMsg->timeStamp;
-  syncLogRecvHeartbeatReply(ths, pMsg, timeDiff);
+  syncLogRecvHeartbeatReply(ths, pMsg, timeDiff, tbuf);
 
   // update last reply time, make decision whether the other node is alive or not
   syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs);
@@ -2335,6 +2571,26 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   SyncLocalCmd* pMsg = pRpcMsg->pCont;
   syncLogRecvLocalCmd(ths, pMsg, "");
 
+  if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) {
+    syncNodeStepDown(ths, pMsg->sdNewTerm);
+
+  } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT) {
+    (void)syncNodeUpdateCommitIndex(ths, pMsg->fcIndex);
+    if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) {
+      sError("vgId:%d, failed to commit raft log since %s. commit index: %" PRId64 "", ths->vgId, terrstr(),
+             ths->commitIndex);
+    }
+  } else {
+    sError("error local cmd");
+  }
+
+  return 0;
+}
+
+int32_t syncNodeOnLocalCmdOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
+  SyncLocalCmd* pMsg = pRpcMsg->pCont;
+  syncLogRecvLocalCmd(ths, pMsg, "");
+
   if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) {
     syncNodeStepDown(ths, pMsg->sdNewTerm);
 
@@ -2362,6 +2618,31 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
 int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) {
   sNTrace(ths, "on client request");
 
+  int32_t code = 0;
+
+  SyncIndex       index = syncLogBufferGetEndIndex(ths->pLogBuf);
+  SyncTerm        term = ths->pRaftStore->currentTerm;
+  SSyncRaftEntry* pEntry = NULL;
+  if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
+    pEntry = syncEntryBuildFromClientRequest(pMsg->pCont, term, index);
+  } else {
+    pEntry = syncEntryBuildFromRpcMsg(pMsg, term, index);
+  }
+
+  if (ths->state == TAOS_SYNC_STATE_LEADER) {
+    if (pRetIndex) {
+      (*pRetIndex) = index;
+    }
+
+    return syncNodeAppend(ths, pEntry);
+  }
+
+  return -1;
+}
+
+int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) {
+  sNTrace(ths, "on client request");
+
   int32_t ret = 0;
   int32_t code = 0;
 
@@ -2385,7 +2666,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn
         if (h) {
           taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
         } else {
-          syncEntryDestory(pEntry);
+          syncEntryDestroy(pEntry);
         }
 
         return -1;
@@ -2408,7 +2689,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn
         if (h) {
           taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
         } else {
-          syncEntryDestory(pEntry);
+          syncEntryDestroy(pEntry);
         }
 
         return -1;
@@ -2443,7 +2724,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn
   if (h) {
     taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
   } else {
-    syncEntryDestory(pEntry);
+    syncEntryDestroy(pEntry);
   }
 
   return ret;
@@ -2457,8 +2738,12 @@ const char* syncStr(ESyncState state) {
       return "candidate";
     case TAOS_SYNC_STATE_LEADER:
       return "leader";
-    default:
+    case TAOS_SYNC_STATE_ERROR:
       return "error";
+    case TAOS_SYNC_STATE_OFFLINE:
+      return "offline";
+    default:
+      return "unknown";
   }
 }
 
@@ -2549,6 +2834,7 @@ bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) {
 }
 
 int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) {
+  ASSERT(false);
   if (beginIndex > endIndex) {
     return 0;
   }
@@ -2662,7 +2948,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde
         if (h) {
           taosLRUCacheRelease(pCache, h, false);
         } else {
-          syncEntryDestory(pEntry);
+          syncEntryDestroy(pEntry);
         }
       }
     }

source/libs/sync/src/syncMessage.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "syncMessage.h"
 #include "syncRaftEntry.h"
+#include "syncRaftStore.h"
 
 int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS,
                          SSyncNode* pNode) {
@@ -153,6 +154,34 @@ int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId) {
   return 0;
 }
 
+int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm,
+                                          SRpcMsg* pRpcMsg) {
+  uint32_t dataLen = pEntry->bytes;
+  uint32_t bytes = sizeof(SyncAppendEntries) + dataLen;
+  pRpcMsg->contLen = bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  if (pRpcMsg->pCont == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  SyncAppendEntries* pMsg = pRpcMsg->pCont;
+  pMsg->bytes = pRpcMsg->contLen;
+  pMsg->msgType = pRpcMsg->msgType = TDMT_SYNC_APPEND_ENTRIES;
+  pMsg->dataLen = dataLen;
+
+  (void)memcpy(pMsg->data, pEntry, dataLen);
+
+  pMsg->prevLogIndex = pEntry->index - 1;
+  pMsg->prevLogTerm = prevLogTerm;
+  pMsg->vgId = pNode->vgId;
+  pMsg->srcId = pNode->myRaftId;
+  pMsg->term = pNode->pRaftStore->currentTerm;
+  pMsg->commitIndex = pNode->commitIndex;
+  pMsg->privateTerm = 0;
+  return 0;
+}
+
 int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId) {
   int32_t bytes = sizeof(SyncHeartbeat);
   pMsg->pCont = rpcMallocCont(bytes);

source/libs/sync/src/syncRaftCfg.c

@@ -395,4 +395,3 @@ int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg) {
   cJSON_Delete(pRoot);
   return 0;
 }
-

source/libs/sync/src/syncRaftEntry.c

@@ -20,7 +20,7 @@
 
 SSyncRaftEntry* syncEntryBuild(int32_t dataLen) {
   int32_t         bytes = sizeof(SSyncRaftEntry) + dataLen;
-  SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes);
+  SSyncRaftEntry* pEntry = taosMemoryCalloc(1, bytes);
   if (pEntry == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
@@ -89,7 +89,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId)
   return pEntry;
 }
 
-void syncEntryDestory(SSyncRaftEntry* pEntry) {
+void syncEntryDestroy(SSyncRaftEntry* pEntry) {
   if (pEntry != NULL) {
     sTrace("free entry: %p", pEntry);
     taosMemoryFree(pEntry);
@@ -265,7 +265,7 @@ static int cmpFn(const void* p1, const void* p2) { return memcmp(p1, p2, sizeof(
 
 static void freeRaftEntry(void* param) {
   SSyncRaftEntry* pEntry = (SSyncRaftEntry*)param;
-  syncEntryDestory(pEntry);
+  syncEntryDestroy(pEntry);
 }
 
 SRaftEntryCache* raftEntryCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) {
@@ -395,7 +395,7 @@ int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) {
       SSkipListNode* pNode = tSkipListIterGet(pIter);
       ASSERT(pNode != NULL);
       SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode);
-      syncEntryDestory(pEntry);
+      syncEntryDestroy(pEntry);
       ++returnCnt;
     }
     tSkipListDestroyIter(pIter);
@@ -424,7 +424,7 @@ int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) {
       ++returnCnt;
       SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode);
 
-      // syncEntryDestory(pEntry);
+      // syncEntryDestroy(pEntry);
       taosRemoveRef(pCache->refMgr, pEntry->rid);
     }
     tSkipListDestroyIter(pIter);

source/libs/sync/src/syncRaftLog.c

@@ -201,9 +201,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
   syncMeta.isWeek = pEntry->isWeak;
   syncMeta.seqNum = pEntry->seqNum;
   syncMeta.term = pEntry->term;
-
   int64_t tsWriteBegin = taosGetTimestampNs();
-  index = walAppendLog(pWal, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
+  index = walAppendLog(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
   int64_t tsWriteEnd = taosGetTimestampNs();
   int64_t tsElapsed = tsWriteEnd - tsWriteBegin;
 
@@ -217,7 +216,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
             pEntry->index, err, err, errStr, sysErr, sysErrStr);
     return -1;
   }
-  pEntry->index = index;
+
+  ASSERT(pEntry->index == index);
 
   sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index,
           TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed);
@@ -344,8 +344,6 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
     const char* sysErrStr = strerror(errno);
     sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
            pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr);
-
-    // ASSERT(0);
   }
 
   // event log
@@ -397,8 +395,6 @@ int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) {
     const char* sysErrStr = strerror(errno);
     sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
            pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr);
-
-    ASSERT(0);
     return -1;
   }
   return 0;

source/libs/sync/src/syncReplication.c

@@ -16,13 +16,11 @@
 #define _DEFAULT_SOURCE
 #include "syncReplication.h"
 #include "syncIndexMgr.h"
+#include "syncPipeline.h"
 #include "syncRaftEntry.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
 
-static int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
-static int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
-
 // TLA+ Spec
 // AppendEntries(i, j) ==
 //    /\ i /= j
@@ -48,7 +46,10 @@ static int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* d
 //                mdest          |-> j])
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 
+int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
+
 int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapshot) {
+  ASSERT(false && "deprecated");
   // next index
   SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId);
 
@@ -117,7 +118,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
   if (h) {
     taosLRUCacheRelease(pCache, h, false);
   } else {
-    syncEntryDestory(pEntry);
+    syncEntryDestroy(pEntry);
   }
 
   // prepare msg
@@ -136,7 +137,29 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
   return 0;
 }
 
-int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
+int32_t syncNodeReplicate(SSyncNode* pNode) {
+  SSyncLogBuffer* pBuf = pNode->pLogBuf;
+  taosThreadMutexLock(&pBuf->mutex);
+  int32_t ret = syncNodeReplicateWithoutLock(pNode);
+  taosThreadMutexUnlock(&pBuf->mutex);
+  return ret;
+}
+
+int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) {
+  if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->replicaNum == 1) {
+    return -1;
+  }
+  for (int32_t i = 0; i < pNode->replicaNum; i++) {
+    if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) {
+      continue;
+    }
+    SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i];
+    (void)syncLogReplMgrReplicateOnce(pMgr, pNode);
+  }
+  return 0;
+}
+
+int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) {
   if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) {
     return -1;
   }
@@ -159,6 +182,14 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
 }
 
 int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
+  SyncAppendEntries* pMsg = pRpcMsg->pCont;
+  int32_t ret = 0;
+  pMsg->destId = *destRaftId;
+  syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg);
+  return 0;
+}
+
+int32_t syncNodeSendAppendEntriesOld(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
   int32_t            ret = 0;
   SyncAppendEntries* pMsg = pRpcMsg->pCont;
   if (pMsg == NULL) {

source/libs/sync/src/syncSnapshot.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "syncSnapshot.h"
 #include "syncIndexMgr.h"
+#include "syncPipeline.h"
 #include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
@@ -273,6 +274,11 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
     return 1;
   }
 
+  char     host[64];
+  uint16_t port;
+  syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port);
+  sInfo("vgId:%d, start snapshot for peer: %s:%d", pSyncNode->vgId, host, port);
+
   code = snapshotSenderStart(pSender);
   if (code != 0) {
     sNError(pSyncNode, "snapshot sender start error");
@@ -372,7 +378,10 @@ int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapsh
 }
 
 int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pPreMsg) {
-  ASSERT(!snapshotReceiverIsStart(pReceiver));
+  if (snapshotReceiverIsStart(pReceiver)) {
+    sWarn("vgId:%d, snapshot receiver has started.", pReceiver->pSyncNode->vgId);
+    return 0;
+  }
 
   pReceiver->start = true;
   pReceiver->ack = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT;
@@ -541,6 +550,10 @@ _START_RECEIVER:
       taosMsleep(10);
     }
 
+    if (snapshotReceiverIsStart(pReceiver)) {
+      snapshotReceiverForceStop(pReceiver);
+    }
+
     snapshotReceiverStart(pReceiver, pMsg);  // set start-time same with sender
   }
 
@@ -738,6 +751,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
 
       } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
         syncNodeOnSnapshotEnd(pSyncNode, pMsg);
+        (void)syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode);
 
       } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) {
         // force close, no response
@@ -869,11 +883,10 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
       // receive ack is finish, close sender
       if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
         snapshotSenderStop(pSender, true);
-
+        SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId);
-        // update next-index
+        if (pMgr) {
-        syncIndexMgrSetIndex(pSyncNode->pNextIndex, &(pMsg->srcId), pMsg->lastIndex + 1);
+            syncLogReplMgrReset(pMgr);
-        syncNodeReplicateOne(pSyncNode, &(pMsg->srcId), false);
+        }
-
         return 0;
       }