Merge remote-tracking branch 'origin/3.0' into fix/coverity.pw

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG d58230c
+        GIT_TAG 4d02980
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/14-reference/05-taosbenchmark.md

@@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js
 
 - **name**: specify the name of the database.
 
-- **drop**: indicate whether to delete the database before inserting. The default is true.
+- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop.
 
 #### Stream processing related configuration parameters
 

docs/examples/python/tmq_example.py

@@ -1,6 +1,58 @@
 import taos
-from taos.tmq import TaosConsumer
+from taos.tmq import *
-consumer = TaosConsumer('topic_ctb_column', group_id='vg2')
+
-for msg in consumer:
+conn = taos.connect()
-    for row in msg:
+
-        print(row)
+print("init")
+conn.execute("drop database if exists py_tmq")
+conn.execute("create database if not exists py_tmq vgroups 2")
+conn.select_db("py_tmq")
+conn.execute(
+    "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)"
+)
+conn.execute("create table if not exists tb1 using stb1 tags(1)")
+conn.execute("create table if not exists tb2 using stb1 tags(2)")
+conn.execute("create table if not exists tb3 using stb1 tags(3)")
+
+print("create topic")
+conn.execute("drop topic if exists topic_ctb_column")
+conn.execute(
+    "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1"
+)
+
+print("build consumer")
+conf = TaosTmqConf()
+conf.set("group.id", "tg2")
+conf.set("td.connect.user", "root")
+conf.set("td.connect.pass", "taosdata")
+conf.set("enable.auto.commit", "true")
+
+
+def tmq_commit_cb_print(tmq, resp, offset, param=None):
+    print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}")
+
+
+conf.set_auto_commit_cb(tmq_commit_cb_print, None)
+tmq = conf.new_consumer()
+
+print("build topic list")
+
+topic_list = TaosTmqList()
+topic_list.append("topic_ctb_column")
+
+print("basic consume loop")
+tmq.subscribe(topic_list)
+
+sub_list = tmq.subscription()
+
+print("subscribed topics: ", sub_list)
+
+while 1:
+    res = tmq.poll(1000)
+    if res:
+        topic = res.get_topic_name()
+        vg = res.get_vgroup_id()
+        db = res.get_db_name()
+        print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}")
+        for row in res:
+            print(row)

docs/zh/12-taos-sql/29-changes.md

@@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明"
 | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
+
+
+## SCHEMALESS 变更
+
+| # | **元素**  | **<div style={{width: 60}}>差异性</div>** | **说明** |
+| - | :------- | :-------- | :------- |
+| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。

docs/zh/14-reference/05-taosbenchmark.md

@@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
 
 - **name** : 数据库名。
 
-- **drop** : 插入前是否删除数据库，默认为 true。
+- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。
 
 #### 流式计算相关配置参数
 

examples/c/demoapi.c

@@ -1,5 +1,17 @@
-// C api call sequence demo
+/*
-// to compile: gcc -o apidemo apidemo.c -ltaos
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -13,23 +25,23 @@
 #include "taos.h"
 
 #define debugPrint(fmt, ...) \
-    do { if (g_args.debug_print || g_args.verbose_print) \
+    do { if (g_args.debug_print || g_args.verbose_print) {\
-      fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0)
+      fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0)
 
 #define warnPrint(fmt, ...) \
     do { fprintf(stderr, "\033[33m"); \
         fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \
-        fprintf(stderr, "\033[0m"); } while(0)
+        fprintf(stderr, "\033[0m"); } while (0)
 
 #define errorPrint(fmt, ...) \
     do { fprintf(stderr, "\033[31m"); \
         fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \
-        fprintf(stderr, "\033[0m"); } while(0)
+        fprintf(stderr, "\033[0m"); } while (0)
 
 #define okPrint(fmt, ...) \
     do { fprintf(stderr, "\033[32m"); \
         fprintf(stderr, "OK: "fmt, __VA_ARGS__); \
-        fprintf(stderr, "\033[0m"); } while(0)
+        fprintf(stderr, "\033[0m"); } while (0)
 
 int64_t g_num_of_tb = 2;
 int64_t g_num_of_rec = 3;
@@ -75,14 +87,15 @@ static void prepare_data(TAOS* taos) {
     taos_free_result(res);
     taosMsleep(100);
     if (taos_select_db(taos, "test")) {
-        errorPrint("%s() LN%d: error no: %d, reason: %s\n",
+        errorPrint("%s() LN%d: taos_select_db() failed\n",
-                __func__, __LINE__, taos_errno(res), taos_errstr(res));
+                __func__, __LINE__);
-        taos_free_result(res);
         return;
     }
 
     char command[1024] = {0};
-    sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));");
+    sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, "
+            "bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, "
+            "city binary(20), dist nchar(20), street binary(20));");
     res = taos_query(taos, command);
     if ((res) && (0 == taos_errno(res))) {
         okPrint("%s created\n", "meters");
@@ -122,12 +135,11 @@ static void prepare_data(TAOS* taos) {
             sprintf(command, "insert into t%"PRId64" "
                     "values(%" PRId64 ", %f, %"PRId64", "
                     "'%c%d', '%s%c%d', '%c%d')",
-                    i, 1650000000000+j, (float)j, j,
+                    i, 1650000000000+j, j * 1.0, j,
                     'a'+(int)j%25, rand(),
                     // "涛思", 'z' - (int)j%25, rand(),
                     "TAOS", 'z' - (int)j%25, rand(),
-                    'b' - (int)j%25, rand()
+                    'b' - (int)j%25, rand());
-                    );
             res = taos_query(taos,  command);
             if ((res) && (0 == taos_errno(res))) {
                 affected = taos_affected_rows(res);
@@ -155,8 +167,7 @@ static void prepare_data(TAOS* taos) {
                 i, 1650000000000+j+1, (float)j, j,
                 'a'+(int)j%25, rand(),
                 "数据", 'z' - (int)j%25, rand(),
-                'b' - (int)j%25, rand()
+                'b' - (int)j%25, rand());
-               );
         res = taos_query(taos,  command);
         if ((res) && (0 == taos_errno(res))) {
             affected = taos_affected_rows(res);
@@ -184,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
     }
 
     if (block) {
-        warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
+        warnPrint("%s", "call taos_fetch_block(), "
+                "don't call taos_fetch_lengths()\n");
         int rows = 0;
         while ((rows = taos_fetch_block(res, &row))) {
             int *lengths = taos_fetch_lengths(res);
@@ -195,7 +207,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                     printf("col%d type is %d, no need get offset\n",
                             f, fields[f].type);
                     for (int64_t c = 0; c < rows; c++) {
-                        switch(fields[f].type) {
+                        switch (fields[f].type) {
                             case TSDB_DATA_TYPE_TIMESTAMP:
                                 if (taos_is_null(res, c, f)) {
                                     printf("col%d, row: %"PRId64" "
@@ -204,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %"PRId64"\n",
                                             f, c,
-                                            *(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
+                                            *(int64_t*)((char*)(row[f])
+                                                +c*sizeof(int64_t)));
                                 }
                                 break;
 
@@ -216,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %d\n",
                                             f, c,
-                                            *(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
+                                            *(int32_t*)((char*)(row[f])
+                                                +c*sizeof(int32_t)));
                                 }
                                 break;
 
@@ -228,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %f\n",
                                             f, c,
-                                            *(float*)((char*)(row[f])+c*sizeof(float)));
+                                            *(float*)((char*)(row[f])
+                                                +c*sizeof(float)));
                                 }
                                 break;
 
@@ -243,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                     if (offsets) {
                         for (int c = 0; c < rows; c++) {
                             if (offsets[c] != -1) {
-                                int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
+                                int length = *(int16_t*)((char*)(row[f])
+                                        + offsets[c]);
                                 char *buf = calloc(1, length + 1);
-                                strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
+                                strncpy(buf, (char *)((char*)(row[f])
-                                printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
+                                            + offsets[c] + 2), length);
+                                printf("row: %d, col: %d, offset: %d, "
+                                        "length: %d, content: %s\n",
                                         c, f, offsets[c], length, buf);
                                 free(buf);
                             } else {
-                                printf("row: %d, col: %d, offset: -1, means content is NULL\n",
+                                printf("row: %d, col: %d, offset: -1, "
+                                        "means content is NULL\n",
                                         c, f);
                             }
                         }
@@ -272,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
             int* lengths = taos_fetch_lengths(res);
             if (lengths) {
                 for (int c = 0; c < num_fields; c++) {
-                    printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n",
+                    printf("row: %"PRId64", col: %d, is_null: %s, "
+                            "length of column %d is %d\n",
                             num_rows, c,
                             taos_is_null(res, num_rows, c)?"True":"False",
                             c, lengths[c]);
@@ -282,7 +302,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                         __func__, __LINE__, tbname);
             }
 
-            num_rows ++;
+            num_rows++;
         }
     }
 
@@ -327,7 +347,8 @@ int main(int argc, char *argv[]) {
 #endif
     TAOS* taos = taos_connect(host, user, passwd, "", 0);
     if (taos == NULL) {
-        printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));
+        printf("\033[31mfailed to connect to db, reason:%s\033[0m\n",
+                taos_errstr(taos));
         exit(1);
     }
 

include/common/tmsg.h

@@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) {
   if (pRebInfo == NULL) {
     return NULL;
   }
-  strcpy(pRebInfo->key, key);
+  tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN);
   pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t));
   if (pRebInfo->lostConsumers == NULL) {
     goto _err;

include/dnode/mnode/mnode.h

@@ -30,9 +30,10 @@ typedef struct SMnode SMnode;
 
 typedef struct {
   int32_t  dnodeId;
-  bool     standby;
   bool     deploy;
-  SReplica replica;
+  int8_t   selfIndex;
+  int8_t   numOfReplicas;
+  SReplica replicas[TSDB_MAX_REPLICA];
   SMsgCb   msgCb;
 } SMnodeOpt;
 

include/libs/sync/sync.h

@@ -211,7 +211,7 @@ typedef struct SSyncInfo {
 
 int32_t     syncInit();
 void        syncCleanUp();
-int64_t     syncOpen(const SSyncInfo* pSyncInfo);
+int64_t     syncOpen(SSyncInfo* pSyncInfo);
 void        syncStart(int64_t rid);
 void        syncStop(int64_t rid);
 int32_t     syncSetStandby(int64_t rid);
@@ -233,7 +233,7 @@ const char* syncStr(ESyncState state);
 bool        syncIsRestoreFinish(int64_t rid);
 int32_t     syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot);
 
-int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg);
+int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg);
 
 // build SRpcMsg, need to call syncPropose with SRpcMsg
 int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg);

include/libs/wal/wal.h

@@ -43,6 +43,7 @@ extern "C" {
 #define WAL_FILE_LEN      (WAL_PATH_LEN + 32)
 #define WAL_MAGIC         0xFAFBFCFDF4F3F2F1ULL
 #define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3)
+#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE)
 
 typedef enum {
   TAOS_WAL_WRITE = 1,

include/os/osEnv.h

@@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace;
 void osDefaultInit();
 void osUpdate();
 void osCleanup();
+
 bool osLogSpaceAvailable();
 bool osDataSpaceAvailable();
 bool osTempSpaceAvailable();
+
+bool osLogSpaceSufficient();
+bool osDataSpaceSufficient();
+bool osTempSpaceSufficient();
+
 void osSetTimezone(const char *timezone);
 void osSetSystemLocale(const char *inLocale, const char *inCharSet);
 

include/util/taoserror.h

@@ -450,6 +450,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_WAL_INVALID_VER               TAOS_DEF_ERROR_CODE(0, 0x1003)
 #define TSDB_CODE_WAL_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x1004)
 #define TSDB_CODE_WAL_LOG_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x1005)
+#define TSDB_CODE_WAL_CHKSUM_MISMATCH           TAOS_DEF_ERROR_CODE(0, 0x1006)
 
 // tfs
 #define TSDB_CODE_FS_INVLD_CFG                  TAOS_DEF_ERROR_CODE(0, 0x2201)

packaging/tools/make_install.sh

@@ -168,7 +168,7 @@ function install_bin() {
 
     ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || :
-    [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || :
+    [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
     [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
@@ -182,21 +182,21 @@ function install_bin() {
 
     ${csudo}chmod 0555 ${install_main_dir}/bin/*
     #Make link
-    [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
+    [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
+    [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || :
+    [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || :
+    [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
+    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
+    [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || :
+    [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || :
+    [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
+    [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
+    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || :
   else
 
     ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || :
-    [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || :
+    [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
     [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
@@ -206,14 +206,14 @@ function install_bin() {
     ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || :
     ${csudo}chmod 0555 ${install_main_dir}/bin/*
     #Make link
-    [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
+    [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
+    [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || :
+    [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || :
+    [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
+    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
-    [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || :
+    [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || :
+    [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || :
-    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
+    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || :
   fi
 }
 
@@ -238,7 +238,7 @@ function install_jemalloc() {
     if [ -f "${binary_dir}/build/lib/libjemalloc.so.2" ]; then
       ${csudo}/usr/bin/install -c -d /usr/local/lib
       ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib
-      ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+      ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so > /dev/null 2>&1
       ${csudo}/usr/bin/install -c -d /usr/local/lib
       [ -f ${binary_dir}/build/lib/libjemalloc.a ] &&
         ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib
@@ -274,8 +274,8 @@ function install_avro() {
     if [ -f "${binary_dir}/build/$1/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then
       ${csudo}/usr/bin/install -c -d /usr/local/$1
       ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.so.23.0.0 /usr/local/$1
-      ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23
+      ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 > /dev/null 2>&1
-      ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so
+      ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so > /dev/null 2>&1
       ${csudo}/usr/bin/install -c -d /usr/local/$1
       [ -f ${binary_dir}/build/$1/libavro.a ] &&
         ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.a /usr/local/$1
@@ -304,11 +304,11 @@ function install_lib() {
       ${install_main_dir}/driver &&
       ${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber}
 
-    ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
+    ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 > /dev/null 2>&1
-    ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
+    ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so > /dev/null 2>&1
     if [ -d "${lib64_link_dir}" ]; then
-        ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1
+        ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 > /dev/null 2>&1
-        ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so
+        ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so > /dev/null 2>&1
     fi
 
     if [ -f ${binary_dir}/build/lib/libtaosws.so ]; then
@@ -316,23 +316,23 @@ function install_lib() {
             ${install_main_dir}/driver &&
             ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||:
 
-        ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
+        ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so > /dev/null 2>&1 || :
     fi
   else
     ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \
       ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
 
     ${csudo}ln -sf ${install_main_dir}/driver/libtaos.${verNumber}.dylib \
-      ${lib_link_dir}/libtaos.1.dylib || :
+      ${lib_link_dir}/libtaos.1.dylib > /dev/null 2>&1 || :
 
-    ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib || :
+    ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib > /dev/null 2>&1 || :
 
     if [ -f ${binary_dir}/build/lib/libtaosws.dylib ]; then
         ${csudo}cp ${binary_dir}/build/lib/libtaosws.dylib \
             ${install_main_dir}/driver &&
             ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.dylib ||:
 
-        ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib || :
+        ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib > /dev/null 2>&1 || :
     fi
   fi
 
@@ -346,6 +346,7 @@ function install_lib() {
 }
 
 function install_header() {
+  ${csudo}mkdir -p ${inc_link_dir}
   ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
   [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||:
   ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
@@ -353,13 +354,13 @@ function install_header() {
 
   if [ -f ${binary_dir}/build/include/taosws.h ]; then
     ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||:
-    ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||:
+    ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h > /dev/null 2>&1 ||:
   fi
 
-  ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
+  ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h > /dev/null 2>&1
-  ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
+  ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h > /dev/null 2>&1
-  ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
+  ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h > /dev/null 2>&1
-  ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
+  ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h > /dev/null 2>&1
 
   ${csudo}chmod 644 ${install_main_dir}/include/*
 }
@@ -374,7 +375,7 @@ function install_config() {
     ${csudo}cp -f ${script_dir}/../cfg/${configFile} \
       ${cfg_install_dir}/${configFile}.${verNumber}
     ${csudo}ln -s ${cfg_install_dir}/${configFile} \
-      ${install_main_dir}/cfg/${configFile}
+      ${install_main_dir}/cfg/${configFile} > /dev/null 2>&1
   else
     ${csudo}cp -f ${script_dir}/../cfg/${configFile} \
       ${cfg_install_dir}/${configFile}.${verNumber}
@@ -395,7 +396,7 @@ function install_taosadapter_config() {
         ${cfg_install_dir}/taosadapter.toml.${verNumber} || :
     [ -f ${cfg_install_dir}/taosadapter.toml ] &&
       ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml \
-        ${install_main_dir}/cfg/taosadapter.toml || :
+        ${install_main_dir}/cfg/taosadapter.toml > /dev/null 2>&1 || :
   else
     if [ -f "${binary_dir}/test/cfg/taosadapter.toml" ]; then
       ${csudo}cp -f ${binary_dir}/test/cfg/taosadapter.toml \
@@ -408,12 +409,12 @@ function install_taosadapter_config() {
 function install_log() {
   ${csudo}rm -rf ${log_dir} || :
   ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
-  ${csudo}ln -s ${log_dir} ${install_main_dir}/log
+  ${csudo}ln -s ${log_dir} ${install_main_dir}/log > /dev/null 2>&1
 }
 
 function install_data() {
   ${csudo}mkdir -p ${data_dir} && ${csudo}chmod 777 ${data_dir}
-  ${csudo}ln -s ${data_dir} ${install_main_dir}/data
+  ${csudo}ln -s ${data_dir} ${install_main_dir}/data > /dev/null 2>&1
 }
 
 function install_connector() {
@@ -533,7 +534,7 @@ function install_taosadapter_service() {
 function install_service_on_launchctl() {
   ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
   ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist
-  ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || :
+  ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
 }
 
 function install_service() {

source/client/src/clientHb.c

@@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
       pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes;
       pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes;
       pTscObj->connId = pRsp->query->connId;
-      tscTrace("conn %d hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes,
+      tscTrace("conn %u hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes,
                pTscObj->pAppInfo->totalDnodes);
 
       if (pRsp->query->killRid) {

source/client/src/clientImpl.c

@@ -186,7 +186,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   STscObj* pTscObj = (*pRequest)->pTscObj;
   if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
                   sizeof((*pRequest)->self))) {
-    tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self,
+    tscError("%" PRIx64 " failed to add to request container, reqId:0x%" PRIu64 ", conn:%" PRIx64 ", %s", (*pRequest)->self,
              (*pRequest)->requestId, pTscObj->id, sql);
 
     taosMemoryFree(param);
@@ -371,7 +371,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
     pInfo->pQnodeList = taosArrayDup(pNodeList);
     taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad);
     tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId,
-             taosArrayGetSize(pInfo->pQnodeList));
+             (int)taosArrayGetSize(pInfo->pQnodeList));
   }
   taosThreadMutexUnlock(&pInfo->qnodeMutex);
 

source/client/src/clientMain.c

@@ -146,7 +146,7 @@ void taos_close(TAOS *taos) {
 
 int taos_errno(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
-    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
     return terrno;
   }
 
@@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) {
     return 0;
   }
 
-  return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL
+  return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code;
-                                                              : ((SRequestObj *)res)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
-    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
     return (const char *)tstrerror(terrno);
   }
 
@@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) {
   if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
-    return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL)
+    return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY)
                                                     : (const char *)tstrerror(pRequest->code);
   }
 }

source/client/src/clientRawBlockWrite.c

@@ -410,6 +410,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
   SDecoder     decoder = {0};
   SVAlterTbReq vAlterTbReq = {0};
   char*        string = NULL;
+  cJSON*       json   = NULL;
 
   // decode
   void*   data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -419,7 +420,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
     goto _exit;
   }
 
-  cJSON* json = cJSON_CreateObject();
+  json = cJSON_CreateObject();
   if (json == NULL) {
     goto _exit;
   }
@@ -524,6 +525,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
   SDecoder     decoder = {0};
   SVDropStbReq req = {0};
   char*        string = NULL;
+  cJSON*       json   = NULL;
 
   // decode
   void*   data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -533,7 +535,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
     goto _exit;
   }
 
-  cJSON* json = cJSON_CreateObject();
+  json = cJSON_CreateObject();
   if (json == NULL) {
     goto _exit;
   }
@@ -556,6 +558,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
   SDecoder         decoder = {0};
   SVDropTbBatchReq req = {0};
   char*            string = NULL;
+  cJSON*           json   = NULL;
 
   // decode
   void*   data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -565,7 +568,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
     goto _exit;
   }
 
-  cJSON* json = cJSON_CreateObject();
+  json = cJSON_CreateObject();
   if (json == NULL) {
     goto _exit;
   }
@@ -684,7 +687,7 @@ end:
 
 static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
   SVDropStbReq req = {0};
-  SDecoder     coder;
+  SDecoder     coder = {0};
   SMDropStbReq pReq = {0};
   int32_t      code = TSDB_CODE_SUCCESS;
   SRequestObj* pRequest = NULL;
@@ -1212,6 +1215,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
   int32_t     code = TSDB_CODE_SUCCESS;
   STableMeta* pTableMeta = NULL;
   SQuery*     pQuery = NULL;
+  SSubmitReq* subReq = NULL;
 
   SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT);
   if (!pRequest) {
@@ -1228,8 +1232,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
   }
 
   SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
-  strcpy(pName.dbname, pRequest->pDb);
+  tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname));
-  strcpy(pName.tname, tbname);
+  tstrncpy(pName.tname, tbname, sizeof(pName.tname));
 
   struct SCatalog* pCatalog = NULL;
   code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
@@ -1278,7 +1282,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
   int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize;
 
   int32_t     totalLen = sizeof(SSubmitReq) + submitLen;
-  SSubmitReq* subReq = taosMemoryCalloc(1, totalLen);
+  subReq = taosMemoryCalloc(1, totalLen);
   SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq));
   void*       blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk));
   STSRow*     rowData = POINTER_SHIFT(blkSchema, schemaLen);
@@ -1352,6 +1356,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
   if (NULL == pQuery) {
     uError("create SQuery error");
     code = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(subReq);
     goto end;
   }
   pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
@@ -1390,6 +1395,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
 end:
   taosMemoryFreeClear(pTableMeta);
   qDestroyQuery(pQuery);
+  taosMemoryFree(subReq);
   return code;
 }
 

source/client/src/clientSml.c

@@ -299,6 +299,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
   for (; i < taosArrayGetSize(cols); i++) {
     SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
     if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) {
+      taosHashCleanup(hashTmp);
       return -1;
     }
   }
@@ -430,7 +431,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
   STableMeta *pTableMeta = NULL;
 
   SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
-  strcpy(pName.dbname, info->pRequest->pDb);
+  tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
 
   SRequestConnInfo conn = {0};
   conn.pTrans = info->taos->pAppInfo->pTransporter;
@@ -874,7 +875,8 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra
   kv->i = ts;
   kv->type = TSDB_DATA_TYPE_TIMESTAMP;
   kv->length = (int16_t)tDataTypes[kv->type].bytes;
-  if (cols) taosArrayPush(cols, &kv);
+  taosArrayPush(cols, &kv);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1009,6 +1011,7 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t *
 
 static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey,
                                   SSmlMsgBuf *msg) {
+  if(!cols) return TSDB_CODE_OUT_OF_MEMORY;
   const char *sql = data;
   size_t      childTableNameLen = strlen(tsSmlChildTableName);
   while (*sql != '\0') {
@@ -1082,7 +1085,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab
     kv->length = valueLen;
     kv->type = TSDB_DATA_TYPE_NCHAR;
 
-    if (cols) taosArrayPush(cols, &kv);
+    taosArrayPush(cols, &kv);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -1370,8 +1373,14 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
   SHashObj *s2 = *(SHashObj **)key2;
   SSmlKv   *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN);
   SSmlKv   *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN);
-  ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP);
+  if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){
-  ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP);
+    uError("smlKvTimeHashCompare kv1");
+    return -1;
+  }
+  if(!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP){
+    uError("smlKvTimeHashCompare kv2");
+    return -1;
+  }
   if (kv1->i < kv2->i) {
     return -1;
   } else if (kv1->i > kv2->i) {
@@ -1735,7 +1744,7 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
   kv->i = tsVal;
   kv->type = TSDB_DATA_TYPE_TIMESTAMP;
   kv->length = (int16_t)tDataTypes[kv->type].bytes;
-  if (cols) taosArrayPush(cols, &kv);
+  taosArrayPush(cols, &kv);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1932,6 +1941,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
 }
 
 static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) {
+  if(!cols) return TSDB_CODE_OUT_OF_MEMORY;
   cJSON *metricVal = cJSON_GetObjectItem(root, "value");
   if (metricVal == NULL) {
     return TSDB_CODE_TSC_INVALID_JSON;
@@ -1941,7 +1951,7 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) {
   if (!kv) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  if (cols) taosArrayPush(cols, &kv);
+  taosArrayPush(cols, &kv);
 
   kv->key = VALUE;
   kv->keyLen = VALUE_LEN;
@@ -1955,7 +1965,9 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) {
 static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey,
                                     SSmlMsgBuf *msg) {
   int32_t ret = TSDB_CODE_SUCCESS;
-
+  if (!pKVs){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
   cJSON *tags = cJSON_GetObjectItem(root, "tags");
   if (tags == NULL || tags->type != cJSON_Object) {
     return TSDB_CODE_TSC_INVALID_JSON;
@@ -1985,14 +1997,14 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN
         return TSDB_CODE_TSC_INVALID_JSON;
       }
       memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
-      strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN);
+      tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN);
       continue;
     }
 
     // add kv to SSmlKv
     SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
     if (!kv) return TSDB_CODE_OUT_OF_MEMORY;
-    if (pKVs) taosArrayPush(pKVs, &kv);
+    taosArrayPush(pKVs, &kv);
 
     // key
     kv->keyLen = keyLen;
@@ -2103,6 +2115,8 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) {
   if (!oneTable) {
     tinfo = smlBuildTableInfo();
     if (!tinfo) {
+      smlDestroyCols(cols);
+      if (info->dataFormat) taosArrayDestroy(cols);
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES);
@@ -2295,7 +2309,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
     SSmlTableInfo *tableData = *oneTable;
 
     SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
-    strcpy(pName.dbname, info->pRequest->pDb);
+    tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
     memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName));
 
     SRequestConnInfo conn = {0};

source/client/src/clientStmt.c

@@ -201,6 +201,9 @@ int32_t stmtCacheBlock(STscStmt* pStmt) {
   }
 
   STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
+  if(!pSrc){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
   STableDataBlocks*  pDst = NULL;
 
   STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc));

source/client/src/clientTmq.c

@@ -233,12 +233,12 @@ void tmq_conf_destroy(tmq_conf_t* conf) {
 
 tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) {
   if (strcmp(key, "group.id") == 0) {
-    strcpy(conf->groupId, value);
+    tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN);
     return TMQ_CONF_OK;
   }
 
   if (strcmp(key, "client.id") == 0) {
-    strcpy(conf->clientId, value);
+    tstrncpy(conf->clientId, value, 256);
     return TMQ_CONF_OK;
   }
 
@@ -452,7 +452,6 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
   int32_t code;
   tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
   if (code < 0) {
-    ASSERT(0);
     return -1;
   }
   void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
@@ -464,15 +463,22 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
   SEncoder encoder;
   tEncoderInit(&encoder, abuf, len);
   tEncodeSTqOffset(&encoder, pOffset);
+  tEncoderClear(&encoder);
 
   // build param
   SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam));
+  if (pParam == NULL) {
+    taosMemoryFree(buf);
+    return -1;
+  }
   pParam->params = pParamSet;
   pParam->pOffset = pOffset;
 
   // build send info
   SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
   if (pMsgSendInfo == NULL) {
+    taosMemoryFree(buf);
+    taosMemoryFree(pParam);
     return -1;
   }
   pMsgSendInfo->msgInfo = (SDataBuf){
@@ -547,6 +553,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm
 
       if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) {
         if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) {
+          tsem_destroy(&pParamSet->rspSem);
+          taosMemoryFree(pParamSet);
           goto FAIL;
         }
         goto HANDLE_RSP;
@@ -565,6 +573,7 @@ HANDLE_RSP:
     tsem_wait(&pParamSet->rspSem);
     code = pParamSet->rspErr;
     tsem_destroy(&pParamSet->rspSem);
+    taosMemoryFree(pParamSet);
     return code;
   } else {
     code = 0;
@@ -587,7 +596,14 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t
 
   SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
   if (pParamSet == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    if (async) {
+      if (automatic) {
+        tmq->commitCb(tmq, code, tmq->commitCbUserParam);
+      } else {
+        userCb(tmq, code, userParam);
+      }
+    }
     return -1;
   }
 
@@ -642,16 +658,6 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t
     code = pParamSet->rspErr;
     tsem_destroy(&pParamSet->rspSem);
     taosMemoryFree(pParamSet);
-  } else {
-    code = 0;
-  }
-
-  if (code != 0 && async) {
-    if (automatic) {
-      tmq->commitCb(tmq, code, tmq->commitCbUserParam);
-    } else {
-      userCb(tmq, code, userParam);
-    }
   }
 
 #if 0
@@ -709,6 +715,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
   int64_t refId = *(int64_t*)param;
   tmq_t*  tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
   if (tmq == NULL) {
+    taosMemoryFree(param);
     return;
   }
   int64_t   consumerId = tmq->consumerId;
@@ -721,6 +728,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
   SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
   if (sendInfo == NULL) {
     taosMemoryFree(pReq);
+    goto OVER;
   }
   sendInfo->msgInfo = (SDataBuf){
       .pData = pReq,
@@ -870,8 +878,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t));
   if (pTmq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(),
+    tscError("setting up new consumer failed since %s, consumer group %s", terrstr(), conf->groupId);
-             pTmq->groupId);
     return NULL;
   }
 
@@ -939,10 +946,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
     return NULL;
   }
 
-  int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
-  *pRefId = pTmq->refId;
-
   if (pTmq->hbBgEnable) {
+    int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
+    *pRefId = pTmq->refId;
     pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer);
   }
 
@@ -966,14 +972,14 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   SCMSubscribeReq req = {0};
   int32_t         code = -1;
 
+  tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz);
+
   req.consumerId = tmq->consumerId;
   tstrncpy(req.clientId, tmq->clientId, 256);
   tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN);
   req.topicNames = taosArrayInit(sz, sizeof(void*));
   if (req.topicNames == NULL) goto FAIL;
 
-  tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz);
-
   for (int32_t i = 0; i < sz; i++) {
     char* topic = taosArrayGetP(container, i);
 
@@ -1061,9 +1067,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   code = 0;
 FAIL:
   if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree);
-  if (code != 0 && buf) {
+  taosMemoryFree(buf);
-    taosMemoryFree(buf);
+
-  }
   return code;
 }
 
@@ -1101,7 +1106,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
       SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM);
       if (pRspWrapper == NULL) {
-        taosMemoryFree(pMsg->pData);
         tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch);
         goto CREATE_MSG_FAIL;
       }

source/common/src/tdataformat.c

@@ -916,6 +916,10 @@ char *tTagValToData(const STagVal *value, bool isJson) {
 }
 
 bool tTagGet(const STag *pTag, STagVal *pTagVal) {
+  if(!pTag || !pTagVal){
+    return false;
+  }
+
   int16_t  lidx = 0;
   int16_t  ridx = pTag->nTag - 1;
   int16_t  midx;

source/dnode/mgmt/mgmt_mnode/inc/mmInt.h

@@ -41,8 +41,8 @@ typedef struct SMnodeMgmt {
 } SMnodeMgmt;
 
 // mmFile.c
-int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed);
+int32_t mmReadFile(const char *path, SMnodeOpt *pOption);
-int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed);
+int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption);
 
 // mmHandle.c
 SArray *mmGetMsgHandles();

source/dnode/mgmt/mgmt_mnode/src/mmFile.c

@@ -16,7 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "mmInt.h"
 
-int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
+int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
   int32_t   code = TSDB_CODE_INVALID_JSON_FORMAT;
   int32_t   len = 0;
   int32_t   maxLen = 4096;
@@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
   char      file[PATH_MAX] = {0};
   TdFilePtr pFile = NULL;
 
-  snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
   pFile = taosOpenFile(file, TD_FILE_READ);
   if (pFile == NULL) {
     code = 0;
@@ -50,38 +50,69 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
     dError("failed to read %s since deployed not found", file);
     goto _OVER;
   }
-  *pDeployed = deployed->valueint;
+  pOption->deploy = deployed->valueint;
 
-  cJSON *id = cJSON_GetObjectItem(root, "id");
+  cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex");
-  if (id) {
+  if (selfIndex) {
-    if (id->type != cJSON_Number) {
+    if (selfIndex->type != cJSON_Number) {
-      dError("failed to read %s since id not found", file);
+      dError("failed to read %s since selfIndex not found", file);
       goto _OVER;
     }
-    if (pReplica) {
+    pOption->selfIndex = selfIndex->valueint;
-      pReplica->id = id->valueint;
-    }
   }
 
-  cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn");
+  cJSON *replicas = cJSON_GetObjectItem(root, "replicas");
-  if (fqdn) {
+  if (replicas) {
-    if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
+    if (replicas->type != cJSON_Array) {
-      dError("failed to read %s since fqdn not found", file);
+      dError("failed to read %s since replicas not found", file);
       goto _OVER;
     }
-    if (pReplica) {
-      tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
-    }
-  }
 
-  cJSON *port = cJSON_GetObjectItem(root, "port");
+    int32_t numOfReplicas = cJSON_GetArraySize(replicas);
-  if (port) {
+    if (numOfReplicas <= 0) {
-    if (port->type != cJSON_Number) {
+      dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas);
-      dError("failed to read %s since port not found", file);
       goto _OVER;
     }
-    if (pReplica) {
+    pOption->numOfReplicas = numOfReplicas;
-      pReplica->port = (uint16_t)port->valueint;
+
+    for (int32_t i = 0; i < numOfReplicas; ++i) {
+      SReplica *pReplica = pOption->replicas + i;
+
+      cJSON *replica = cJSON_GetArrayItem(replicas, i);
+      if (replica == NULL) break;
+
+      cJSON *id = cJSON_GetObjectItem(replica, "id");
+      if (id) {
+        if (id->type != cJSON_Number) {
+          dError("failed to read %s since id not found", file);
+          goto _OVER;
+        }
+        if (pReplica) {
+          pReplica->id = id->valueint;
+        }
+      }
+
+      cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn");
+      if (fqdn) {
+        if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
+          dError("failed to read %s since fqdn not found", file);
+          goto _OVER;
+        }
+        if (pReplica) {
+          tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
+        }
+      }
+
+      cJSON *port = cJSON_GetObjectItem(replica, "port");
+      if (port) {
+        if (port->type != cJSON_Number) {
+          dError("failed to read %s since port not found", file);
+          goto _OVER;
+        }
+        if (pReplica) {
+          pReplica->port = (uint16_t)port->valueint;
+        }
+      }
     }
   }
 
@@ -92,18 +123,18 @@ _OVER:
   if (root != NULL) cJSON_Delete(root);
   if (pFile != NULL) taosCloseFile(&pFile);
   if (code == 0) {
-    dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+    dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy);
   }
 
   terrno = code;
   return code;
 }
 
-int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) {
+int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
   char file[PATH_MAX] = {0};
   char realfile[PATH_MAX] = {0};
-  snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
+  snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
-  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
 
   TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
@@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
   char   *content = taosMemoryCalloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  if (pReplica != NULL && pReplica->id > 0) {
+  if (pOption->deploy && pOption->numOfReplicas > 0) {
-    len += snprintf(content + len, maxLen - len, "  \"id\": %d,\n", pReplica->id);
+    len += snprintf(content + len, maxLen - len, "  \"selfIndex\": %d,\n", pOption->selfIndex);
-    len += snprintf(content + len, maxLen - len, "  \"fqdn\": \"%s\",\n", pReplica->fqdn);
+    len += snprintf(content + len, maxLen - len, "  \"replicas\": [{\n");
-    len += snprintf(content + len, maxLen - len, "  \"port\": %u\n,", pReplica->port);
+
+    for (int32_t i = 0; i < pOption->numOfReplicas; ++i) {
+      const SReplica *pReplica = pOption->replicas + i;
+      if (pReplica != NULL && pReplica->id > 0) {
+        len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pReplica->id);
+        len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pReplica->fqdn);
+        len += snprintf(content + len, maxLen - len, "    \"port\": %u\n", pReplica->port);
+      }
+      if (i < pOption->numOfReplicas - 1) {
+        len += snprintf(content + len, maxLen - len, "  },{\n");
+      } else {
+        len += snprintf(content + len, maxLen - len, "  }],\n");
+      }
+    }
   }
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d\n", deployed);
+  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d\n", pOption->deploy);
   len += snprintf(content + len, maxLen - len, "}\n");
 
   taosWriteFile(pFile, content, len);
@@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
     return -1;
   }
 
-  dDebug("successed to write %s, deployed:%d", realfile, deployed);
+  dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy);
   return 0;
 }

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
     return -1;
   }
 
-  if (createReq.replica != 1) {
+  SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1};
+  memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas));
+  for (int32_t i = 0; i < option.numOfReplicas; ++i) {
+    if (createReq.replicas[i].id == pInput->pData->dnodeId) {
+      option.selfIndex = i;
+    }
+  }
+
+  if (option.selfIndex == -1) {
     terrno = TSDB_CODE_INVALID_OPTION;
-    dGError("failed to create mnode since %s", terrstr());
+    dGError("failed to create mnode since %s, selfIndex is -1", terrstr());
     return -1;
   }
 
-  bool deployed = true;
+  if (mmWriteFile(pInput->path, &option) != 0) {
-
-  SMnodeMgmt mgmt = {0};
-  mgmt.path = pInput->path;
-  mgmt.name = pInput->name;
-  if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) {
     dGError("failed to write mnode file since %s", terrstr());
     return -1;
   }
@@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
     return -1;
   }
 
-  bool deployed = false;
+  SMnodeOpt option = {.deploy = false};
-
+  if (mmWriteFile(pInput->path, &option) != 0) {
-  SMnodeMgmt mgmt = {0};
-  mgmt.path = pInput->path;
-  mgmt.name = pInput->name;
-  if (mmWriteFile(&mgmt, NULL, deployed) != 0) {
     dGError("failed to write mnode file since %s", terrstr());
     return -1;
   }
@@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
-  // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_mnode/src/mmInt.c

@@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
 }
 
 static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
-  SMnodeMgmt mgmt = {0};
+  SMnodeOpt option = {0};
-  mgmt.path = pInput->path;
+  if (mmReadFile(pInput->path, &option) != 0) {
-  if (mmReadFile(&mgmt, NULL, required) != 0) {
     return -1;
   }
 
-  if (!(*required)) {
+  if (!option.deploy) {
     *required = mmDeployRequired(pInput);
+  } else {
+    *required = true;
   }
 
   return 0;
 }
 
 static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
-  pOption->standby = false;
   pOption->deploy = true;
   pOption->msgCb = pMgmt->msgCb;
   pOption->dnodeId = pMgmt->pData->dnodeId;
-  pOption->replica.id = 1;
+  pOption->selfIndex = 0;
-  pOption->replica.port = tsServerPort;
+  pOption->numOfReplicas = 1;
-  tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
+  pOption->replicas[0].id = 1;
+  pOption->replicas[0].port = tsServerPort;
+  tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
 }
 
-static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) {
+static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
-  pOption->standby = false;
   pOption->deploy = false;
   pOption->msgCb = pMgmt->msgCb;
   pOption->dnodeId = pMgmt->pData->dnodeId;
-  if (pReplica->id > 0) {
-    pOption->standby = true;
-    pOption->replica = *pReplica;
-  }
 }
 
 static void mmClose(SMnodeMgmt *pMgmt) {
@@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
   pMgmt->msgCb.mgmt = pMgmt;
   taosThreadRwlockInit(&pMgmt->lock, NULL);
 
-  bool     deployed = false;
+  SMnodeOpt option = {0};
-  SReplica replica = {0};
+  if (mmReadFile(pMgmt->path, &option) != 0) {
-  if (mmReadFile(pMgmt, &replica, &deployed) != 0) {
     dError("failed to read file since %s", terrstr());
     mmClose(pMgmt);
     return -1;
   }
 
-  SMnodeOpt option = {0};
+  if (!option.deploy) {
-  if (!deployed) {
     dInfo("mnode start to deploy");
     pMgmt->pData->dnodeId = 1;
     mmBuildOptionForDeploy(pMgmt, pInput, &option);
   } else {
     dInfo("mnode start to open");
-    mmBuildOptionForOpen(pMgmt, &replica, &option);
+    mmBuildOptionForOpen(pMgmt, &option);
   }
 
   pMgmt->pMnode = mndOpen(pMgmt->path, &option);
@@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
   }
   tmsgReportStartup("mnode-worker", "initialized");
 
-  if (!deployed || replica.id > 0) {
+  if (option.numOfReplicas > 0) {
-    deployed = true;
+    option.deploy = true;
-    if (mmWriteFile(pMgmt, NULL, deployed) != 0) {
+    option.numOfReplicas = 0;
+    if (mmWriteFile(pMgmt->path, &option) != 0) {
       dError("failed to write mnode file since %s", terrstr());
       return -1;
     }

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -51,26 +51,14 @@ static int32_t dmInitMonitor() {
 
 static bool dmCheckDiskSpace() {
   osUpdate();
-  if (!osDataSpaceAvailable()) {
+  if (!osDataSpaceSufficient()) {
-    dError("free disk size: %f GB, too little, require %f GB at least at least , quit",
+    dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-           (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
-           (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-    terrno = TSDB_CODE_NO_AVAIL_DISK;
-    return false;
   }
-  if (!osLogSpaceAvailable()) {
+  if (!osLogSpaceSufficient()) {
-    dError("free disk size: %f GB, too little, require %f GB at least at least, quit",
+    dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-           (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
-           (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-    terrno = TSDB_CODE_NO_AVAIL_DISK;
-    return false;
   }
-  if (!osTempSpaceAvailable()) {
+  if (!osTempSpaceSufficient()) {
-    dError("free disk size: %f GB, too little, require %f GB at least at least, quit",
+    dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-           (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
-           (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
-    terrno = TSDB_CODE_NO_AVAIL_DISK;
-    return false;
   }
   return true;
 }

source/dnode/mnode/impl/inc/mndInt.h

@@ -87,12 +87,13 @@ typedef struct {
 typedef struct {
   tsem_t   syncSem;
   int64_t  sync;
-  SReplica replica;
   int32_t  errCode;
   int32_t  transId;
   SRWLatch lock;
-  int8_t   standby;
   int8_t   leaderTransferFinish;
+  int8_t   selfIndex;
+  int8_t   numOfReplicas;
+  SReplica replicas[TSDB_MAX_REPLICA];
 } SSyncMgmt;
 
 typedef struct {
@@ -130,11 +131,10 @@ typedef struct SMnode {
 void    mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
 int64_t mndGenerateUid(const char *name, int32_t len);
 
-int32_t mndAcquireRpcRef(SMnode *pMnode);
+void mndSetRestored(SMnode *pMnode, bool restored);
-void    mndReleaseRpcRef(SMnode *pMnode);
+bool mndGetRestored(SMnode *pMnode);
-void    mndSetRestore(SMnode *pMnode, bool restored);
+void mndSetStop(SMnode *pMnode);
-void    mndSetStop(SMnode *pMnode);
+bool mndGetStop(SMnode *pMnode);
-bool    mndGetStop(SMnode *pMnode);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndSync.h

@@ -24,7 +24,7 @@ extern "C" {
 
 int32_t mndInitSync(SMnode *pMnode);
 void    mndCleanupSync(SMnode *pMnode);
-bool    mndIsMaster(SMnode *pMnode);
+bool    mndIsLeader(SMnode *pMnode);
 int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId);
 void    mndSyncStart(SMnode *pMnode);
 void    mndSyncStop(SMnode *pMnode);

source/dnode/mnode/impl/src/mndDb.c

@@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
   }
 
   while (numOfRows < rowsCapacity) {
-    pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus);
+    pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true);
     if (pShow->pIter == NULL) break;
 
     if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {

source/dnode/mnode/impl/src/mndDnode.c

@@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
         goto _OVER;
       } else {
         pDnode->accessTimes++;
-        mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
+        mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
       }
     }
 
@@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
   }
 
   pDnode->lastAccessTime = curMs;
+  pDnode->accessTimes++;
   code = 0;
 
 _OVER:

source/dnode/mnode/impl/src/mndMain.c

@@ -43,6 +43,37 @@
 #include "mndUser.h"
 #include "mndVgroup.h"
 
+static inline int32_t mndAcquireRpc(SMnode *pMnode) {
+  int32_t code = 0;
+  taosThreadRwlockRdlock(&pMnode->lock);
+  if (pMnode->stopped) {
+    terrno = TSDB_CODE_APP_NOT_READY;
+    code = -1;
+  } else if (!mndIsLeader(pMnode)) {
+    code = -1;
+  } else {
+#if 1
+    atomic_add_fetch_32(&pMnode->rpcRef, 1);
+#else
+    int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
+    mTrace("mnode rpc is acquired, ref:%d", ref);
+#endif
+  }
+  taosThreadRwlockUnlock(&pMnode->lock);
+  return code;
+}
+
+static inline void mndReleaseRpc(SMnode *pMnode) {
+  taosThreadRwlockRdlock(&pMnode->lock);
+#if 1
+  atomic_sub_fetch_32(&pMnode->rpcRef, 1);
+#else
+  int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
+  mTrace("mnode rpc is released, ref:%d", ref);
+#endif
+  taosThreadRwlockUnlock(&pMnode->lock);
+}
+
 static void *mndBuildTimerMsg(int32_t *pContLen) {
   SMTimerReq timerReq = {0};
 
@@ -201,7 +232,7 @@ static int32_t mndInitWal(SMnode *pMnode) {
 
   pMnode->pWal = walOpen(path, &cfg);
   if (pMnode->pWal == NULL) {
-    mError("failed to open wal since %s", terrstr());
+    mError("failed to open wal since %s. wal:%s", terrstr(), path);
     return -1;
   }
 
@@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) {
 static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
   pMnode->msgCb = pOption->msgCb;
   pMnode->selfDnodeId = pOption->dnodeId;
-  pMnode->syncMgmt.replica = pOption->replica;
+  pMnode->syncMgmt.selfIndex = pOption->selfIndex;
-  pMnode->syncMgmt.standby = pOption->standby;
+  pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas;
+  memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas));
 }
 
 SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
@@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) {
       mError("failed to deploy sdb while start mnode");
       return -1;
     }
-    mndSetRestore(pMnode, true);
+    mndSetRestored(pMnode, true);
   }
 
   grantReset(pMnode, TSDB_GRANT_ALL, 0);
@@ -570,23 +602,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
       pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) {
     return 0;
   }
-  if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
+  if (mndAcquireRpc(pMsg->info.node) == 0) return 0;
   if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
       pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
       pMsg->msgType == TDMT_MND_UPTIME_TIMER) {
     return -1;
   }
 
-  SEpSet epSet = {0};
+  SEpSet  epSet = {0};
-  mndGetMnodeEpSet(pMsg->info.node, &epSet);
+  SMnode *pMnode = pMsg->info.node;
+  mndGetMnodeEpSet(pMnode, &epSet);
 
   const STraceId *trace = &pMsg->info.traceId;
-  mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(),
+  mDebug(
-         TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
+      "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s "
+      "numOfEps:%d inUse:%d",
+      pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync),
+      syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
 
   if (epSet.numOfEps > 0) {
     for (int32_t i = 0; i < epSet.numOfEps; ++i) {
-      mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
+      mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
     }
 
     int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
@@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
 
   mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
   int32_t code = (*fp)(pMsg);
-  mndReleaseRpcRef(pMnode);
+  mndReleaseRpc(pMnode);
 
   if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
     mGTrace("msg:%p, won't response immediately since in progress", pMsg);
@@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
 
 int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
                           SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
-  if (mndAcquireRpcRef(pMnode) != 0) return -1;
+  if (mndAcquireRpc(pMnode) != 0) return -1;
 
   SSdb   *pSdb = pMnode->pSdb;
   int64_t ms = taosGetTimestampMs();
@@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
   pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc));
   if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
       pStbInfo->stbs == NULL) {
-    mndReleaseRpcRef(pMnode);
+    mndReleaseRpc(pMnode);
     return -1;
   }
 
@@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
     pGrantInfo->timeseries_total = INT32_MAX;
   }
 
-  mndReleaseRpcRef(pMnode);
+  mndReleaseRpc(pMnode);
   return 0;
 }
 
@@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
   return 0;
 }
 
-int32_t mndAcquireRpcRef(SMnode *pMnode) {
+void mndSetRestored(SMnode *pMnode, bool restored) {
-  int32_t code = 0;
-  taosThreadRwlockRdlock(&pMnode->lock);
-  if (pMnode->stopped) {
-    mTrace("mnode not running");
-    terrno = TSDB_CODE_APP_NOT_READY;
-    code = -1;
-  } else if (!mndIsMaster(pMnode)) {
-    mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored);
-    code = -1;
-  } else {
-    int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
-    // mTrace("mnode rpc is acquired, ref:%d", ref);
-  }
-  taosThreadRwlockUnlock(&pMnode->lock);
-  return code;
-}
-
-void mndReleaseRpcRef(SMnode *pMnode) {
-  taosThreadRwlockRdlock(&pMnode->lock);
-  int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
-  // mTrace("mnode rpc is released, ref:%d", ref);
-  taosThreadRwlockUnlock(&pMnode->lock);
-}
-
-void mndSetRestore(SMnode *pMnode, bool restored) {
   if (restored) {
     taosThreadRwlockWrlock(&pMnode->lock);
     pMnode->restored = true;

source/dnode/mnode/impl/src/mndMnode.c

@@ -36,6 +36,7 @@ static int32_t  mndProcessAlterMnodeReq(SRpcMsg *pReq);
 static int32_t  mndProcessDropMnodeReq(SRpcMsg *pReq);
 static int32_t  mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextMnode(SMnode *pMnode, void *pIter);
+static void     mndReloadSyncConfig(SMnode *pMnode);
 
 int32_t mndInitMnode(SMnode *pMnode) {
   SSdbTable table = {
@@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
   }
 
   pObj->state = TAOS_SYNC_STATE_ERROR;
+  mndReloadSyncConfig(pSdb->pMnode);
   return 0;
 }
 
@@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
 static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) {
   mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
   pOld->updateTime = pNew->updateTime;
+  mndReloadSyncConfig(pSdb->pMnode);
+
   return 0;
 }
 
@@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
     if (pIter == NULL) break;
 
     if (pObj->id == pMnode->selfDnodeId) {
-      if (mndIsMaster(pMnode)) {
+      if (mndIsLeader(pMnode)) {
         pEpSet->inUse = pEpSet->numOfEps;
       } else {
         pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes;
@@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
   if (pEpSet->numOfEps == 0) {
     syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet);
   }
+
+  if (pEpSet->inUse >= pEpSet->numOfEps) {
+    pEpSet->inUse = 0;
+  }
 }
 
 static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
@@ -274,13 +282,72 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod
   return 0;
 }
 
+static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
+  int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
+  void   *pReq = taosMemoryMalloc(contLen);
+  tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
+
+  STransAction action = {
+      .epSet = *pCreateEpSet,
+      .pCont = pReq,
+      .contLen = contLen,
+      .msgType = TDMT_DND_CREATE_MNODE,
+      .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED,
+  };
+
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(pReq);
+    return -1;
+  }
+  return 0;
+}
+
+static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
+  int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
+  void   *pReq = taosMemoryMalloc(contLen);
+  tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
+
+  STransAction action = {
+      .epSet = *pAlterEpSet,
+      .pCont = pReq,
+      .contLen = contLen,
+      .msgType = TDMT_MND_ALTER_MNODE,
+      .acceptableCode = 0,
+  };
+
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(pReq);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
+  int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq);
+  void   *pReq = taosMemoryMalloc(contLen);
+  tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq);
+
+  STransAction action = {
+      .epSet = *pDroprEpSet,
+      .pCont = pReq,
+      .contLen = contLen,
+      .msgType = TDMT_DND_DROP_MNODE,
+      .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
+  };
+
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(pReq);
+    return -1;
+  }
+  return 0;
+}
+
 static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
   SSdb            *pSdb = pMnode->pSdb;
   void            *pIter = NULL;
   int32_t          numOfReplicas = 0;
-  SDAlterMnodeReq  alterReq = {0};
   SDCreateMnodeReq createReq = {0};
-  SEpSet           alterEpset = {0};
   SEpSet           createEpset = {0};
 
   while (1) {
@@ -288,75 +355,25 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
     pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
     if (pIter == NULL) break;
 
-    alterReq.replicas[numOfReplicas].id = pMObj->id;
+    createReq.replicas[numOfReplicas].id = pMObj->id;
-    alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
+    createReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
-    memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
+    memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-
-    alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
-    memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-    if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
-      alterEpset.inUse = numOfReplicas;
-    }
 
     numOfReplicas++;
     sdbRelease(pSdb, pMObj);
   }
 
-  alterReq.replica = numOfReplicas + 1;
+  createReq.replica = numOfReplicas + 1;
-  alterReq.replicas[numOfReplicas].id = pDnode->id;
+  createReq.replicas[numOfReplicas].id = pDnode->id;
-  alterReq.replicas[numOfReplicas].port = pDnode->port;
+  createReq.replicas[numOfReplicas].port = pDnode->port;
-  memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
+  memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-
-  alterEpset.numOfEps = numOfReplicas + 1;
-  alterEpset.eps[numOfReplicas].port = pDnode->port;
-  memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-
-  createReq.replica = 1;
-  createReq.replicas[0].id = pDnode->id;
-  createReq.replicas[0].port = pDnode->port;
-  memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
 
+  createEpset.inUse = 0;
   createEpset.numOfEps = 1;
   createEpset.eps[0].port = pDnode->port;
   memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
 
-  {
+  if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
-    int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq);
-    void   *pReq = taosMemoryMalloc(contLen);
-    tSerializeSDCreateMnodeReq(pReq, contLen, &createReq);
-
-    STransAction action = {
-        .epSet = createEpset,
-        .pCont = pReq,
-        .contLen = contLen,
-        .msgType = TDMT_DND_CREATE_MNODE,
-        .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED,
-    };
-
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
-      return -1;
-    }
-  }
-
-  {
-    int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
-    void   *pReq = taosMemoryMalloc(contLen);
-    tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
-
-    STransAction action = {
-        .epSet = alterEpset,
-        .pCont = pReq,
-        .contLen = contLen,
-        .msgType = TDMT_MND_ALTER_MNODE,
-        .acceptableCode = 0,
-    };
-
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
-      return -1;
-    }
-  }
 
   return 0;
 }
@@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
   mndTransSetSerial(pTrans);
   mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
 
+  if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
   if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
   if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
-  if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
   if (mndTransAppendNullLog(pTrans) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
@@ -459,107 +476,28 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO
 }
 
 static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
-  SSdb           *pSdb = pMnode->pSdb;
+  SSdb          *pSdb = pMnode->pSdb;
-  void           *pIter = NULL;
+  void          *pIter = NULL;
-  int32_t         numOfReplicas = 0;
+  int32_t        numOfReplicas = 0;
-  SDAlterMnodeReq alterReq = {0};
+  SDDropMnodeReq dropReq = {0};
-  SDDropMnodeReq  dropReq = {0};
+  SEpSet         dropEpSet = {0};
-  SSetStandbyReq  standbyReq = {0};
-  SEpSet          alterEpset = {0};
-  SEpSet          dropEpSet = {0};
-
-  while (1) {
-    SMnodeObj *pMObj = NULL;
-    pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
-    if (pIter == NULL) break;
-    if (pMObj->id == pObj->id) {
-      sdbRelease(pSdb, pMObj);
-      continue;
-    }
-
-    alterReq.replicas[numOfReplicas].id = pMObj->id;
-    alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
-    memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-
-    alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
-    memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-    if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
-      alterEpset.inUse = numOfReplicas;
-    }
-
-    numOfReplicas++;
-    sdbRelease(pSdb, pMObj);
-  }
-
-  alterReq.replica = numOfReplicas;
-  alterEpset.numOfEps = numOfReplicas;
 
   dropReq.dnodeId = pDnode->id;
   dropEpSet.numOfEps = 1;
   dropEpSet.eps[0].port = pDnode->port;
   memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
 
-  standbyReq.dnodeId = pDnode->id;
+  int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE);
-  standbyReq.standby = 1;
+  if (totalMnodes == 2) {
-
+    mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
-  {
+    if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
-    int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead);
+    if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
-    void   *pReq = taosMemoryMalloc(contLen);
+  } else if (totalMnodes == 3) {
-    tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq);
+    mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
-    SMsgHead *pHead = pReq;
+    if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
-    pHead->contLen = htonl(contLen);
+    if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
-    pHead->vgId = htonl(MNODE_HANDLE);
+  } else {
-
+    return -1;
-    STransAction action = {
-        .epSet = dropEpSet,
-        .pCont = pReq,
-        .contLen = contLen,
-        .msgType = TDMT_SYNC_SET_MNODE_STANDBY,
-        .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
-    };
-
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
-      return -1;
-    }
-  }
-
-  {
-    int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
-    void   *pReq = taosMemoryMalloc(contLen);
-    tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
-
-    STransAction action = {
-        .epSet = alterEpset,
-        .pCont = pReq,
-        .contLen = contLen,
-        .msgType = TDMT_MND_ALTER_MNODE,
-        .acceptableCode = 0,
-    };
-
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
-      return -1;
-    }
-  }
-
-  {
-    int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq);
-    void   *pReq = taosMemoryMalloc(contLen);
-    tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq);
-
-    STransAction action = {
-        .epSet = dropEpSet,
-        .pCont = pReq,
-        .contLen = contLen,
-        .msgType = TDMT_DND_DROP_MNODE,
-        .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
-    };
-
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
-      return -1;
-    }
   }
 
   return 0;
@@ -567,7 +505,6 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
 
 int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
   if (pObj == NULL) return 0;
-  if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
   if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
   if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1;
   if (mndTransAppendNullLog(pTrans) != 0) return -1;
@@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
   int64_t    curMs = taosGetTimestampMs();
 
   while (numOfRows < rows) {
-    pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus);
+    pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true);
     if (pShow->pIter == NULL) break;
 
     cols = 0;
@@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) {
 }
 
 static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
+#if 1
+  return 0;
+#else
   SMnode         *pMnode = pReq->info.node;
   SDAlterMnodeReq alterReq = {0};
 
@@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
     return -1;
   }
 
+  SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
+  memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
+  for (int32_t i = 0; i < option.numOfReplicas; ++i) {
+    if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
+      option.selfIndex = i;
+    }
+  }
+
+  if (option.selfIndex == -1) {
+    mInfo("alter mnode not processed since selfIndex is -1", terrstr());
+    return 0;
+  }
+
+  if (mndWriteFile(pMnode->path, &option) != 0) {
+    mError("failed to write mnode file since %s", terrstr());
+    return -1;
+  }
+
   SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
   for (int32_t i = 0; i < alterReq.replica; ++i) {
     SNodeInfo *pNode = &cfg.nodeInfo[i];
     tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn));
     pNode->nodePort = alterReq.replicas[i].port;
-    if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i;
+    if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
+      cfg.myIndex = i;
+    }
   }
 
   if (cfg.myIndex == -1) {
     mError("failed to alter mnode since myindex is -1");
     return -1;
   } else {
-    mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex);
+    mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
     for (int32_t i = 0; i < alterReq.replica; ++i) {
       SNodeInfo *pNode = &cfg.nodeInfo[i];
       mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
     }
   }
 
-  mInfo("trans:-1, sync reconfig will be proposed");
+  int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
-
-  SSyncMgmt *pMgmt = &pMnode->syncMgmt;
-  pMgmt->standby = 0;
-  int32_t code = syncReconfig(pMgmt->sync, &cfg);
   if (code != 0) {
-    mError("trans:-1, failed to propose sync reconfig since %s", terrstr());
+    mError("failed to sync reconfig since %s", terrstr());
-    return code;
   } else {
-    pMgmt->errCode = 0;
+    mInfo("alter mnode sync success");
-    taosWLockLatch(&pMgmt->lock);
+  }
-    pMgmt->transId = -1;
+
-    taosWUnLockLatch(&pMgmt->lock);
+  return code;
-    tsem_wait(&pMgmt->syncSem);
+#endif
-    mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode));
+}
-    terrno = pMgmt->errCode;
+
-    return pMgmt->errCode;
+static void mndReloadSyncConfig(SMnode *pMnode) {
+  SSdb      *pSdb = pMnode->pSdb;
+  SMnodeObj *pObj = NULL;
+  ESdbStatus objStatus = 0;
+  void      *pIter = NULL;
+  bool       hasUpdatingMnode = false;
+  SSyncCfg   cfg = {.myIndex = -1};
+
+  while (1) {
+    pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false);
+    if (pIter == NULL) break;
+    if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) {
+      mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus));
+      hasUpdatingMnode = true;
+    }
+
+    if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) {
+      SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum];
+      tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn));
+      pNode->nodePort = pObj->pDnode->port;
+      if (pObj->pDnode->id == pMnode->selfDnodeId) {
+        cfg.myIndex = cfg.replicaNum;
+      }
+      cfg.replicaNum++;
+    }
+
+    sdbReleaseLock(pSdb, pObj, false);
+  }
+
+  if (cfg.myIndex == -1) {
+    mInfo("vgId:1, mnode not reload since selfIndex is -1");
+    return;
+  }
+
+  if (!mndGetRestored(pMnode)) {
+    mInfo("vgId:1, mnode not reload since restore not finished");
+    return;
+  }
+
+  if (hasUpdatingMnode) {
+    mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
+    for (int32_t i = 0; i < cfg.replicaNum; ++i) {
+      SNodeInfo *pNode = &cfg.nodeInfo[i];
+      mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
+    }
+
+    int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
+    if (code != 0) {
+      mError("vgId:1, failed to reconfig mnode sync since %s", terrstr());
+    } else {
+      mInfo("vgId:1, reconfig mnode sync success");
+    }
   }
 }

source/dnode/mnode/impl/src/mndScheduler.c

@@ -416,7 +416,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
       }
 
       SStreamTask* pTask = tNewSStreamTask(pStream->uid);
-      if (pInnerTask == NULL) {
+      if (pTask == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         sdbRelease(pSdb, pVgroup);
         qDestroyQueryPlan(pPlan);

source/dnode/mnode/impl/src/mndSync.c

@@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) {
   if (!pMnode->deploy) {
     mInfo("vgId:1, sync restore finished, and will handle outstanding transactions");
     mndTransPullup(pMnode);
-    mndSetRestore(pMnode, true);
+    mndSetRestored(pMnode, true);
   } else {
     mInfo("vgId:1, sync restore finished");
   }
@@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) {
   snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP);
   syncInfo.pWal = pMnode->pWal;
   syncInfo.pFsm = mndSyncMakeFsm(pMnode);
-  syncInfo.isStandBy = pMgmt->standby;
   syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT;
 
-  mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby);
+  mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas);
-  if (pMgmt->standby || pMgmt->replica.id > 0) {
+  SSyncCfg *pCfg = &syncInfo.syncCfg;
-    SSyncCfg *pCfg = &syncInfo.syncCfg;
+  pCfg->replicaNum = pMgmt->numOfReplicas;
-    pCfg->replicaNum = 1;
+  pCfg->myIndex = pMgmt->selfIndex;
-    pCfg->myIndex = 0;
+  for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) {
-    SNodeInfo *pNode = &pCfg->nodeInfo[0];
+    SNodeInfo *pNode = &pCfg->nodeInfo[i];
-    tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn));
+    tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
-    pNode->nodePort = pMgmt->replica.port;
+    pNode->nodePort = pMgmt->replicas[i].port;
-    mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort);
+    mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort);
   }
 
   tsem_init(&pMgmt->syncSem, 0, 0);
@@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) {
   setPingTimerMS(pMgmt->sync, 5000);
   setElectTimerMS(pMgmt->sync, 3000);
   setHeartbeatTimerMS(pMgmt->sync, 500);
-  /*
-    setElectTimerMS(pMgmt->sync, 600);
-    setHeartbeatTimerMS(pMgmt->sync, 300);
-  */
 
   mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
   return 0;
@@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) {
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
   syncSetMsgCb(pMgmt->sync, &pMnode->msgCb);
   syncStart(pMgmt->sync);
-  mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby);
+  mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync);
 }
 
 void mndSyncStop(SMnode *pMnode) {
@@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) {
   taosWUnLockLatch(&pMnode->syncMgmt.lock);
 }
 
-bool mndIsMaster(SMnode *pMnode) {
+bool mndIsLeader(SMnode *pMnode) {
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
 
   if (!syncIsReady(pMgmt->sync)) {
@@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) {
     return false;
   }
 
-  if (!pMnode->restored) {
+  if (!mndGetRestored(pMnode)) {
     terrno = TSDB_CODE_APP_NOT_READY;
     return false;
   }

source/dnode/mnode/impl/src/mndTrans.c

@@ -53,7 +53,7 @@ static bool    mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
-static bool    mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
+static bool    mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); }
 
 static void    mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
 static int32_t mndProcessTransTimer(SRpcMsg *pReq);

source/dnode/mnode/sdb/inc/sdb.h

@@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey);
  * @param pObj The object of the row.
  */
 void sdbRelease(SSdb *pSdb, void *pObj);
+void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock);
 
 /**
  * @brief Traverse a sdb table
@@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj);
  * @return void* The next iterator of the table.
  */
 void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj);
-void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status);
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock);
 
 /**
  * @brief Cancel a traversal

source/dnode/mnode/sdb/src/sdbHash.c

@@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) {
   taosThreadRwlockUnlock(pLock);
 }
 
-void sdbRelease(SSdb *pSdb, void *pObj) {
+void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) {
   if (pObj == NULL) return;
 
   SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow));
   if (pRow->type >= SDB_MAX) return;
 
   TdThreadRwlock *pLock = &pSdb->locks[pRow->type];
-  taosThreadRwlockWrlock(pLock);
+  if (lock) {
+    taosThreadRwlockWrlock(pLock);
+  }
 
   int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
   sdbPrintOper(pSdb, pRow, "release");
@@ -342,9 +344,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
     sdbFreeRow(pSdb, pRow, true);
   }
 
-  taosThreadRwlockUnlock(pLock);
+  if (lock) {
+    taosThreadRwlockUnlock(pLock);
+  }
 }
 
+void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); }
+
 void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
   *ppObj = NULL;
 
@@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
   return ppRow;
 }
 
-void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) {
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) {
   *ppObj = NULL;
 
   SHashObj *hash = sdbGetHash(pSdb, type);
   if (hash == NULL) return NULL;
 
   TdThreadRwlock *pLock = &pSdb->locks[type];
-  taosThreadRwlockRdlock(pLock);
+  if (lock) {
+    taosThreadRwlockRdlock(pLock);
+  }
 
   SSdbRow **ppRow = taosHashIterate(hash, pIter);
   while (ppRow != NULL) {
@@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat
     *status = pRow->status;
     break;
   }
-  taosThreadRwlockUnlock(pLock);
+  if (lock) {
+    taosThreadRwlockUnlock(pLock);
+  }
 
   return ppRow;
 }

source/dnode/vnode/src/inc/tq.h

@@ -115,7 +115,6 @@ typedef struct {
 
 typedef struct {
   SMqDataRsp     dataRsp;
-  SMqRspHead     rspHead;
   char           subKey[TSDB_SUBSCRIBE_KEY_LEN];
   SRpcHandleInfo pInfo;
 } STqPushEntry;
@@ -183,6 +182,7 @@ int32_t         tqOffsetCommitFile(STqOffsetStore* pStore);
 
 // tqSink
 void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
+void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
 
 // tqOffset
 char*   tqOffsetBuildFName(const char* path, int32_t ver);

source/dnode/vnode/src/meta/metaQuery.c

@@ -675,7 +675,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
   SSchemaWrapper *pSchemaWrapper = &schema;
 
   tDecoderInit(&dc, pData, nData);
-  tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
+  (void)tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
   tDecoderClear(&dc);
   tdbFree(pData);
 

source/dnode/vnode/src/meta/metaTable.c

@@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema
     return -1;
   }
 
-  strcpy(pMetaRsp->tbName, tbName);
+  strncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN);
   pMetaRsp->numOfColumns = pSchema->nCols;
   pMetaRsp->tableType = TSDB_NORMAL_TABLE;
   pMetaRsp->sversion = pSchema->version;
@@ -693,6 +693,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
     if (iCol >= pSchema->nCols) break;
     pColumn = &pSchema->pSchema[iCol];
 
+    ASSERT(pAlterTbReq->colName);
     if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break;
     iCol++;
   }
@@ -816,6 +817,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   const void *pData = NULL;
   int         nData = 0;
 
+  if (pAlterTbReq->tagName == NULL) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
   // search name index
   ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
   if (ret < 0) {
@@ -938,6 +944,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
     metaUpdateTagIdx(pMeta, &ctbEntry);
   }
 
+  ASSERT(ctbEntry.ctbEntry.pTags);
   SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid};
   tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags,
               ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
@@ -946,7 +953,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
 
   tDecoderClear(&dc1);
   tDecoderClear(&dc2);
-  if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
+  taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
   if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
   if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
   tdbTbcClose(pTbDbc);
@@ -1196,8 +1203,8 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   SMetaEntry     stbEntry = {0};
   STagIdxKey    *pTagIdxKey = NULL;
   int32_t        nTagIdxKey;
-  const SSchema *pTagColumn;       // = &stbEntry.stbEntry.schema.pSchema[0];
+  const SSchema *pTagColumn;
-  const void    *pTagData = NULL;  //
+  const void    *pTagData = NULL;
   int32_t        nTagData = 0;
   SDecoder       dc = {0};
   int32_t        ret = 0;

source/dnode/vnode/src/tq/tq.c

@@ -192,7 +192,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
     return -1;
   }
 
-  memcpy(buf, &pPushEntry->rspHead, sizeof(SMqRspHead));
+  memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead));
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
 
@@ -215,7 +215,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
   tFormatOffset(buf1, 80, &pRsp->reqOffset);
   tFormatOffset(buf2, 80, &pRsp->rspOffset);
   tqDebug("vgId:%d, from consumer:%" PRId64 ", (epoch %d) push rsp, block num: %d, reqOffset:%s, rspOffset:%s",
-          TD_VID(pTq->pVnode), pPushEntry->rspHead.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2);
+          TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2);
 
   return 0;
 }
@@ -560,9 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
         memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN);
         dataRsp.withTbName = 0;
         memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp));
-        pPushEntry->rspHead.consumerId = consumerId;
+        pPushEntry->dataRsp.head.consumerId = consumerId;
-        pPushEntry->rspHead.epoch = reqEpoch;
+        pPushEntry->dataRsp.head.epoch = reqEpoch;
-        pPushEntry->rspHead.mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
+        pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
         taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*));
         tqDebug("tmq poll: consumer %ld, subkey %s, vg %d save handle to push mgr", consumerId, pHandle->subKey,
                 TD_VID(pTq->pVnode));
@@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) {
     pTask->smaSink.smaSink = smaHandleRes;
   } else if (pTask->outputType == TASK_OUTPUT__TABLE) {
     pTask->tbSink.vnode = pTq->pVnode;
-    pTask->tbSink.tbSinkFunc = tqTableSink;
+    pTask->tbSink.tbSinkFunc = tqTableSink1;
 
     ASSERT(pTask->tbSink.pSchemaWrapper);
     ASSERT(pTask->tbSink.pSchemaWrapper->pSchema);
@@ -1010,16 +1010,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
 
       if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
         qError("stream task input del failed, task id %d", pTask->taskId);
+
+        taosFreeQitem(pRefBlock);
         continue;
       }
+
       if (streamSchedExec(pTask) < 0) {
         qError("stream task launch failed, task id %d", pTask->taskId);
         continue;
       }
+
     } else {
       streamTaskInputFail(pTask);
     }
   }
+
   int32_t ref = atomic_sub_fetch_32(pRef, 1);
   ASSERT(ref >= 0);
   if (ref == 0) {

source/dnode/vnode/src/tq/tqSink.c

@@ -284,6 +284,212 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
   return ret;
 }
 
+void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
+  const SArray*   pBlocks = (const SArray*)data;
+  SVnode*         pVnode = (SVnode*)vnode;
+  int64_t         suid = pTask->tbSink.stbUid;
+  char*           stbFullName = pTask->tbSink.stbFullName;
+  STSchema*       pTSchema = pTask->tbSink.pTSchema;
+  SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;
+
+  int32_t blockSz = taosArrayGetSize(pBlocks);
+  bool    createTb = true;
+
+  SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
+  if (!tagArray) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return;
+  }
+
+  tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz);
+  for (int32_t i = 0; i < blockSz; i++) {
+    SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+    if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
+      SBatchDeleteReq deleteReq = {0};
+      deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq));
+      deleteReq.suid = suid;
+      tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq);
+
+      int32_t len;
+      int32_t code;
+      tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code);
+      if (code < 0) {
+        //
+        ASSERT(0);
+      }
+      SEncoder encoder;
+      void*    serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
+      void*    abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
+      tEncoderInit(&encoder, abuf, len);
+      tEncodeSBatchDeleteReq(&encoder, &deleteReq);
+      tEncoderClear(&encoder);
+      taosArrayDestroy(deleteReq.deleteReqs);
+
+      ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId;
+
+      SRpcMsg msg = {
+          .msgType = TDMT_VND_BATCH_DEL,
+          .pCont = serializedDeleteReq,
+          .contLen = len + sizeof(SMsgHead),
+      };
+      if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+        rpcFreeCont(serializedDeleteReq);
+        tqDebug("failed to put delete req into write-queue since %s", terrstr());
+      }
+    } else {
+      SVCreateTbReq createTbReq = {0};
+
+      // set const
+      createTbReq.flags = 0;
+      createTbReq.type = TSDB_CHILD_TABLE;
+      createTbReq.ctb.suid = suid;
+
+      // set super table name
+      SName name = {0};
+      tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+      createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name));  // strdup(stbFullName);
+
+      // set tag content
+      taosArrayClear(tagArray);
+      STagVal tagVal = {
+          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
+          .type = TSDB_DATA_TYPE_UBIGINT,
+          .i64 = (int64_t)pDataBlock->info.groupId,
+      };
+      taosArrayPush(tagArray, &tagVal);
+      createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
+
+      STag* pTag = NULL;
+      tTagNew(tagArray, 1, false, &pTag);
+      if (pTag == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        taosArrayDestroy(tagArray);
+        tdDestroySVCreateTbReq(&createTbReq);
+        return;
+      }
+      createTbReq.ctb.pTag = (uint8_t*)pTag;
+
+      // set tag name
+      SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
+      char    tagNameStr[TSDB_COL_NAME_LEN] = {0};
+      strcpy(tagNameStr, "group_id");
+      taosArrayPush(tagName, tagNameStr);
+      createTbReq.ctb.tagName = tagName;
+
+      // set table name
+      if (pDataBlock->info.parTbName[0]) {
+        createTbReq.name = strdup(pDataBlock->info.parTbName);
+      } else {
+        createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
+      }
+
+      int32_t schemaLen;
+      int32_t code;
+      tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
+      if (code < 0) {
+        tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        return;
+      }
+
+      // save schema str
+      void* schemaStr = taosMemoryMalloc(schemaLen);
+      if (schemaStr == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        return;
+      }
+
+      SEncoder encoder = {0};
+      tEncoderInit(&encoder, schemaStr, schemaLen);
+      code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
+      if (code < 0) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        tdDestroySVCreateTbReq(&createTbReq);
+        taosArrayDestroy(tagArray);
+        tEncoderClear(&encoder);
+        taosMemoryFree(schemaStr);
+        return;
+      }
+      tEncoderClear(&encoder);
+      tdDestroySVCreateTbReq(&createTbReq);
+
+      int32_t cap = sizeof(SSubmitReq);
+
+      int32_t rows = pDataBlock->info.rows;
+      int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+
+      cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen;
+
+      SSubmitReq* ret = rpcMallocCont(cap);
+      ret->header.vgId = pVnode->config.vgId;
+      ret->length = sizeof(SSubmitReq);
+      ret->numOfBlocks = htonl(1);
+
+      SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq));
+
+      blkHead->numOfRows = htonl(pDataBlock->info.rows);
+      blkHead->sversion = htonl(pTSchema->version);
+      blkHead->suid = htobe64(suid);
+      // uid is assigned by vnode
+      blkHead->uid = 0;
+      blkHead->schemaLen = 0;
+
+      tqDebug("tq sink, convert block %d, rows: %d", i, rows);
+
+      int32_t dataLen = 0;
+      void*   blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
+      STSRow* rowData = blkSchema;
+      if (createTb) {
+        memcpy(blkSchema, schemaStr, schemaLen);
+        blkHead->schemaLen = htonl(schemaLen);
+        rowData = POINTER_SHIFT(blkSchema, schemaLen);
+      }
+
+      taosMemoryFree(schemaStr);
+
+      for (int32_t j = 0; j < rows; j++) {
+        SRowBuilder rb = {0};
+        tdSRowInit(&rb, pTSchema->version);
+        tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen);
+        tdSRowResetBuf(&rb, rowData);
+
+        for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
+          const STColumn*  pColumn = &pTSchema->columns[k];
+          SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k);
+          if (colDataIsNull_s(pColData, j)) {
+            tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k);
+          } else {
+            void* colData = colDataGetData(pColData, j);
+            tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k);
+          }
+        }
+        tdSRowEnd(&rb);
+        int32_t rowLen = TD_ROW_LEN(rowData);
+        rowData = POINTER_SHIFT(rowData, rowLen);
+        dataLen += rowLen;
+      }
+      blkHead->dataLen = htonl(dataLen);
+
+      ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen;
+      ret->length = htonl(ret->length);
+
+      SRpcMsg msg = {
+          .msgType = TDMT_VND_SUBMIT,
+          .pCont = ret,
+          .contLen = ntohl(ret->length),
+      };
+
+      if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+        rpcFreeCont(ret);
+        tqDebug("failed to put into write-queue since %s", terrstr());
+      }
+    }
+  }
+  taosArrayDestroy(tagArray);
+}
+
 void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
   const SArray*   pRes = (const SArray*)data;
   SVnode*         pVnode = (SVnode*)vnode;

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD
     if (code) goto _err;
   }
 
+_err:
   if (aDelData) {
     taosArrayDestroy(aDelData);
   }
-_err:
   return code;
 }
 
@@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
 
   // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
   SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
-  if (code) goto _err;
 
   *pDelIdx = *pIdx;
 
+_err:
   if (pDelIdxArray) {
     taosArrayDestroy(pDelIdxArray);
   }
-_err:
   return code;
 }
 
@@ -599,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
        * &state->blockIdx);
        */
       state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ);
-      if (code) goto _err;
-
       if (!state->pBlockIdx) {
         goto _next_fileset;
       }
@@ -898,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
     if (code) goto _err;
 
     code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
-    if (code) goto _err;
+    if (code) {
+      tsdbDelFReaderClose(&pDelFReader);
+      goto _err;
+    }
 
     code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline);
-    if (code) goto _err;
+    if (code) {
+      tsdbDelFReaderClose(&pDelFReader);
+      goto _err;
+    }
 
     tsdbDelFReaderClose(&pDelFReader);
   } else {
@@ -1231,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
 _err:
   nextRowIterClose(&iter);
   taosMemoryFreeClear(pTSchema);
+  *ppLastArray = NULL;
+  taosArrayDestroy(pColArray);
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1045,7 +1045,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
   STsdb     *pTsdb = pCommitter->pTsdb;
   SMemTable *pMemTable = pTsdb->imem;
 
-  ASSERT(eno == 0);
+  ASSERT(eno == 0 &&
+    "tsdbCommit failure"
+    "Restart taosd");
 
   code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
   TSDB_CHECK_CODE(code, lino, _exit);

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -819,7 +819,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
       nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1);
       if (nRef == 0) {
         tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname);
-        taosRemoveFile(fname);
+        (void)taosRemoveFile(fname);
         taosMemoryFree(fSet.pHeadF);
       }
     } else {
@@ -961,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
     nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1);
     if (nRef == 0) {
       tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname);
-      taosRemoveFile(fname);
+      (void)taosRemoveFile(fname);
       taosMemoryFree(pSetOld->pHeadF);
     }
 
@@ -1114,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
     ASSERT(nRef >= 0);
     if (nRef == 0) {
       tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
-      taosRemoveFile(fname);
+      (void)taosRemoveFile(fname);
       taosMemoryFree(pSet->pHeadF);
     }
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
     goto _err;
   }
 
+  ASSERT(pPool != NULL);
   // do delete
   SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData));
   if (pDelData == NULL) {
@@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
   SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
   int8_t     maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
 
+  ASSERT(pPool != NULL);
   pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
   if (pTbData == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN
 
   // node
   level = tsdbMemSkipListRandLevel(&pTbData->sl);
+  ASSERT(pPool != NULL);
   pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
   if (pNode == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -133,8 +133,15 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
       id.uid = pIter->uid;
     }
 
-    tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
+    code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _exit;
+    }
+
     code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _exit;
+    }
 
     double el = (taosGetTimestampUs() - st) / 1000.0;
     pInfo->elapsedTime += el;
@@ -144,9 +151,6 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
               ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s",
               pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el,
               idStr);
-    if (code != TSDB_CODE_SUCCESS) {
-      goto _exit;
-    }
 
     pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk;
     tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr);
@@ -459,8 +463,8 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) {
 }
 
 bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
-  int32_t code = 0;
   int32_t step = pIter->backward ? -1 : 1;
+  terrno = TSDB_CODE_SUCCESS;
 
   // no qualified last file block in current file, no need to fetch row
   if (pIter->pSttBlk == NULL) {
@@ -469,6 +473,10 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
 
   int32_t     iBlockL = pIter->iSttBlk;
   SBlockData *pBlockData = loadLastBlock(pIter, idStr);
+  if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) {
+    goto _exit;
+  }
+
   pIter->iRow += step;
 
   while (1) {
@@ -494,11 +502,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
   pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow);
 
 _exit:
-  if (code != TSDB_CODE_SUCCESS) {
+  return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL);
-    terrno = code;
-  }
-
-  return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL);
 }
 
 SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -340,7 +340,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
     pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader));
     if (pIter->pLastBlockReader == NULL) {
       int32_t code = TSDB_CODE_OUT_OF_MEMORY;
-      tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr);
+      tsdbError("failed to prepare the last block iterator, code:%s %s", tstrerror(code), pReader->idStr);
       return code;
     }
   }
@@ -646,7 +646,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
 
   double el = (taosGetTimestampUs() - st) / 1000.0;
   tsdbDebug(
-      "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
+      "load block of %"PRIzu" tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
       "time:%.2f ms %s",
       numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el,
       pReader->idStr);
@@ -1515,6 +1515,11 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader*
 
   taosMemoryFree(pReader->pMemSchema);
   int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema);
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    return NULL;
+  }
+
   return pReader->pMemSchema;
 }
 
@@ -2676,9 +2681,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
     int8_t  level = 0;
     int8_t  precision = pVnode->config.tsdbCfg.precision;
     int64_t now = taosGetTimestamp(precision);
-    int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI)   ? 1
+    int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI)   ? 1L
-                                             : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000
+                                             : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
-                                                                                        : 1000000);
+                                                                                        : 1000000L);
 
     for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
       SRetention* pRetention = retentions + level;
@@ -3085,7 +3090,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p
     doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader);
 
     tRowMerge(&merge, piRow);
-    doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader);
+    doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader);
   }
 
   int32_t code = tRowMergerGetRow(&merge, pTSRow);
@@ -3372,14 +3377,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
   if (pCond->suid != 0) {
     pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1);
     if (pReader->pSchema == NULL) {
-      tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1,
+      tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr);
-                pReader->idStr);
     }
   } else if (taosArrayGetSize(pTableList) > 0) {
     STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
     pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1);
     if (pReader->pSchema == NULL) {
-      tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr);
+      tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);
     }
   }
 
@@ -3444,7 +3448,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
   return code;
 
 _err:
-  tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr);
+  tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
   return code;
 }
 
@@ -3733,7 +3737,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
 }
 
 int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
-  if (isEmptyQueryTimeWindow(&pReader->window)) {
+  if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) {
     return TSDB_CODE_SUCCESS;
   }
 

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -124,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
   }
 
   // check
-  if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
+  if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
     code = TSDB_CODE_FILE_CORRUPTED;
     goto _exit;
   }
@@ -843,7 +843,8 @@ _err:
 // SDataFReader ====================================================
 int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
   int32_t       code = 0;
-  SDataFReader *pReader;
+  int32_t       lino = 0;
+  SDataFReader *pReader = NULL;
   int32_t       szPage = pTsdb->pVnode->config.tsdbPageSize;
   char          fname[TSDB_FILENAME_LEN];
 
@@ -851,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
   pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
   if (pReader == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
   pReader->pTsdb = pTsdb;
   pReader->pSet = pSet;
@@ -859,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
   // head
   tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
   code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD);
-  if (code) goto _err;
+  TSDB_CHECK_CODE(code, lino, _exit);
 
   // data
   tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
   code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD);
-  if (code) goto _err;
+  TSDB_CHECK_CODE(code, lino, _exit);
 
   // sma
   tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
   code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD);
-  if (code) goto _err;
+  TSDB_CHECK_CODE(code, lino, _exit);
 
   // stt
   for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
     tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
     code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]);
-    if (code) goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  *ppReader = pReader;
+_exit:
-  return code;
+  if (code) {
+    *ppReader = NULL;
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
 
-_err:
+    if (pReader) {
-  tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+      for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]);
-  *ppReader = NULL;
+      tsdbCloseFile(&pReader->pSmaFD);
+      tsdbCloseFile(&pReader->pDataFD);
+      tsdbCloseFile(&pReader->pHeadFD);
+      taosMemoryFree(pReader);
+    }
+  } else {
+    *ppReader = pReader;
+  }
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -445,13 +445,14 @@ _err:
 
 int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) {
   int32_t          code = 0;
+  int32_t          lino = 0;
   STsdbSnapReader* pReader = NULL;
 
   // alloc
   pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
   if (pReader == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
   pReader->pTsdb = pTsdb;
   pReader->sver = sver;
@@ -461,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
   code = taosThreadRwlockRdlock(&pTsdb->rwLock);
   if (code) {
     code = TAOS_SYSTEM_ERROR(code);
-    goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
 
   code = tsdbFSRef(pTsdb, &pReader->fs);
   if (code) {
     taosThreadRwlockUnlock(&pTsdb->rwLock);
-    goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
 
   code = taosThreadRwlockUnlock(&pTsdb->rwLock);
   if (code) {
     code = TAOS_SYSTEM_ERROR(code);
-    goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
 
   // data
@@ -485,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
       pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
       if (pIter->aBlockIdx == NULL) {
         code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _err;
+        TSDB_CHECK_CODE(code, lino, _exit);
       }
     } else {
       pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
       if (pIter->aSttBlk == NULL) {
         code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _err;
+        TSDB_CHECK_CODE(code, lino, _exit);
       }
     }
 
     code = tBlockDataCreate(&pIter->bData);
-    if (code) goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
 
   code = tBlockDataCreate(&pReader->bData);
-  if (code) goto _err;
+  TSDB_CHECK_CODE(code, lino, _exit);
 
   // del
   pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
   if (pReader->aDelIdx == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
   pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
   if (pReader->aDelData == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
+    TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
+_exit:
-  *ppReader = pReader;
+  if (code) {
-  return code;
+    tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code),
+              pTsdb->path);
+    *ppReader = NULL;
 
-_err:
+    if (pReader) {
-  tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
+      taosArrayDestroy(pReader->aDelData);
-            tstrerror(code));
+      taosArrayDestroy(pReader->aDelIdx);
-  *ppReader = NULL;
+      tBlockDataDestroy(&pReader->bData, 1);
+      tsdbFSDestroy(&pReader->fs);
+      taosMemoryFree(pReader);
+    }
+  } else {
+    *ppReader = pReader;
+    tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
+  }
   return code;
 }
 

source/dnode/vnode/src/vnd/vnodeBufPool.c

@@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) {
 void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
   SVBufPoolNode *pNode;
   void          *p = NULL;
+  ASSERT(pPool != NULL);
+
   taosThreadSpinLock(&pPool->lock);
   if (pPool->node.size >= pPool->ptr - pPool->node.data + size) {
     // allocate from the anchor node

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) {
 
 int vnodeShouldCommit(SVnode *pVnode) {
   if (pVnode->inUse) {
-    return pVnode->inUse->size > pVnode->inUse->node.size;
+    return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size);
   }
   return false;
 }
@@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
   data = NULL;
 
   if (vnodeEncodeInfo(pInfo, &data) < 0) {
+    vError("failed to encode json info.");
     return -1;
   }
 
@@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
   }
 
   if (taosWriteFile(pFile, data, strlen(data)) < 0) {
-    vError("failed to write info file:%s data:%s", fname, terrstr());
+    vError("failed to write info file:%s error:%s", fname, terrstr());
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
@@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) {
     snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path);
   }
   if (vnodeSaveInfo(dir, &info) < 0) {
-    ASSERT(0);
+    vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
   walBeginSnapshot(pVnode->pWal, pVnode->state.applied);
 
   // preCommit
   // smaSyncPreCommit(pVnode->pSma);
-  if (smaAsyncPreCommit(pVnode->pSma) < 0) {
+  if(smaAsyncPreCommit(pVnode->pSma) < 0){
-    ASSERT(0);
+    vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
 
@@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) {
 
   // commit each sub-system
   if (metaCommit(pVnode->pMeta) < 0) {
-    ASSERT(0);
+    vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
 
   if (VND_IS_RSMA(pVnode)) {
     if (smaAsyncCommit(pVnode->pSma) < 0) {
-      ASSERT(0);
+      vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
       return -1;
     }
 
     if (tsdbCommit(VND_RSMA0(pVnode)) < 0) {
-      ASSERT(0);
+      vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
       return -1;
     }
     if (tsdbCommit(VND_RSMA1(pVnode)) < 0) {
-      ASSERT(0);
+      vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno));
       return -1;
     }
     if (tsdbCommit(VND_RSMA2(pVnode)) < 0) {
-      ASSERT(0);
+      vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
       return -1;
     }
   } else {
     if (tsdbCommit(pVnode->pTsdb) < 0) {
-      ASSERT(0);
+      vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
       return -1;
     }
   }
 
   if (tqCommit(pVnode->pTq) < 0) {
-    ASSERT(0);
+    vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
   // walCommit (TODO)
 
   // commit info
   if (vnodeCommitInfo(dir, &info) < 0) {
-    ASSERT(0);
+    vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
 
@@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) {
   // postCommit
   // smaSyncPostCommit(pVnode->pSma);
   if (smaAsyncPostCommit(pVnode->pSma) < 0) {
-    ASSERT(0);
+    vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
 

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
 
   pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg));
   if (pVnode->pWal == NULL) {
-    vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno));
+    vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir);
     goto _err;
   }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -41,16 +41,28 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
       int32_t nReqs;
 
       tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
-      tStartDecode(&dc);
+      if (tStartDecode(&dc) < 0) {
+        code = TSDB_CODE_INVALID_MSG;
+        return code;
+      }
 
-      tDecodeI32v(&dc, &nReqs);
+      if (tDecodeI32v(&dc, &nReqs) < 0) {
+        code = TSDB_CODE_INVALID_MSG;
+        goto _err;
+      }
       for (int32_t iReq = 0; iReq < nReqs; iReq++) {
         tb_uid_t uid = tGenIdPI64();
         char    *name = NULL;
         tStartDecode(&dc);
 
-        tDecodeI32v(&dc, NULL);
+        if (tDecodeI32v(&dc, NULL) < 0) {
-        tDecodeCStr(&dc, &name);
+          code = TSDB_CODE_INVALID_MSG;
+          return code;
+        }
+        if (tDecodeCStr(&dc, &name) < 0) {
+          code = TSDB_CODE_INVALID_MSG;
+          return code;
+        }
         *(int64_t *)(dc.data + dc.pos) = uid;
         *(int64_t *)(dc.data + dc.pos + 8) = ctime;
 
@@ -81,10 +93,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
           char *name = NULL;
 
           tDecoderInit(&dc, pBlock->data, msgIter.schemaLen);
-          tStartDecode(&dc);
+          if (tStartDecode(&dc) < 0) {
+            code = TSDB_CODE_INVALID_MSG;
+            return code;
+          }
 
-          tDecodeI32v(&dc, NULL);
+          if (tDecodeI32v(&dc, NULL) < 0) {
-          tDecodeCStr(&dc, &name);
+            code = TSDB_CODE_INVALID_MSG;
+            return code;
+          }
+          if (tDecodeCStr(&dc, &name) < 0) {
+            code = TSDB_CODE_INVALID_MSG;
+            return code;
+          }
 
           uid = metaGetTableEntryUidByName(pVnode->pMeta, name);
           if (uid == 0) {
@@ -148,6 +169,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
   int32_t len;
   int32_t ret;
 
+  if (!pVnode->inUse) {
+    terrno = TSDB_CODE_VND_NOT_SYNCED;
+    vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr());
+    return -1;
+  }
+
   vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
          version);
 
@@ -268,10 +295,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
   _do_commit:
     vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
     // commit current change
-    vnodeCommit(pVnode);
+    if (vnodeCommit(pVnode) < 0) {
+      vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno));
+      goto _err;
+    }
 
     // start a new one
-    vnodeBegin(pVnode);
+    if (vnodeBegin(pVnode) < 0) {
+      vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno));
+      goto _err;
+    }
   }
 
   return 0;

source/libs/executor/src/dataInserter.c

@@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
     }
 
     pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows;
-    qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows,
+    qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows,
            pInserter->submitRes.affectedRows);
 
     tFreeSSubmitRsp(pInserter->submitRes.pRsp);

source/libs/executor/src/executil.c

@@ -1138,7 +1138,7 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i
   s.bytes = bytes;
   s.slotId = slotId;
   s.precision = precision;
-  strncpy(s.name, name, tListLen(s.name));
+  tstrncpy(s.name, name, tListLen(s.name));
 
   return s;
 }
@@ -1201,14 +1201,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) {
     pExp->base.resSchema =
         createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName);
 
-    pExp->pExpr->_function.functionId = pFuncNode->funcId;
+    tExprNode* pExprNode = pExp->pExpr;
-    pExp->pExpr->_function.pFunctNode = pFuncNode;
+
+    pExprNode->_function.functionId = pFuncNode->funcId;
+    pExprNode->_function.pFunctNode = pFuncNode;
+
+    tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName));
 
-    strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
-            tListLen(pExp->pExpr->_function.functionName));
 #if 1
     // todo refactor: add the parameter for tbname function
-    if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
+    const char* name = "tbname";
+    int32_t len = strlen(name);
+
+    if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) &&
+        pExprNode->_function.functionName[len] == 0) {
+
       pFuncNode->pParameterList = nodesMakeList();
       ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
       SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
@@ -1359,7 +1366,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
           fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
         } else {
           char* udfName = pExpr->pExpr->_function.pFunctNode->functionName;
-          strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN);
+          tstrncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN);
           fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet);
         }
         pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);

source/libs/executor/src/executor.c

@@ -616,7 +616,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
 
   int32_t nOptrWithVal = 0;
   int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
-  if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) {
+  if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) {
     taosMemoryFreeClear(*pOutput);
     *len = 0;
   }
@@ -701,10 +701,10 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
   return 0;
 }
 
-int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo mtInfo) {
+int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) {
   memset(pCond, 0, sizeof(SQueryTableDataCond));
   pCond->order = TSDB_ORDER_ASC;
-  pCond->numOfCols = mtInfo.schema->nCols;
+  pCond->numOfCols = pMtInfo->schema->nCols;
   pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
   if (pCond->colList == NULL) {
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
@@ -712,15 +712,15 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
   }
 
   pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
-  pCond->suid = mtInfo.suid;
+  pCond->suid = pMtInfo->suid;
   pCond->type = TIMEWINDOW_RANGE_CONTAINED;
   pCond->startVersion = -1;
   pCond->endVersion = sContext->snapVersion;
 
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-    pCond->colList[i].type = mtInfo.schema->pSchema[i].type;
+    pCond->colList[i].type = pMtInfo->schema->pSchema[i].type;
-    pCond->colList[i].bytes = mtInfo.schema->pSchema[i].bytes;
+    pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes;
-    pCond->colList[i].colId = mtInfo.schema->pSchema[i].colId;
+    pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -844,7 +844,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList);
     if (mtInfo.uid == 0) return 0;  // no data
 
-    initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo);
+    initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
     pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
     pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
     taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0});
@@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
     pTaskInfo->streamInfo.schema = mtInfo.schema;
 
-    qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts);
+    qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts);
   } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
     SStreamRawScanInfo* pInfo = pOperator->info;
     SSnapContext*       sContext = pInfo->sContext;
     if (setForSnapShot(sContext, pOffset->uid) != 0) {
-      qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid);
+      qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
       return -1;
     }
-    qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid);
+    qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts);
   } else if (pOffset->type == TMQ_OFFSET__LOG) {
     SStreamRawScanInfo* pInfo = pOperator->info;
     tsdbReaderClose(pInfo->dataReader);

source/libs/executor/src/executorimpl.c

@@ -895,33 +895,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI
 }
 #endif
 
-static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
-#if 0
-  SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
-  uint32_t        status = BLK_DATA_NOT_LOAD;
-
-  int32_t numOfOutput = 0;  // pTableScanInfo->numOfOutput;
-  for (int32_t i = 0; i < numOfOutput; ++i) {
-    int32_t functionId = pCtx[i].functionId;
-    int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId;
-
-    // group by + first/last should not apply the first/last block filter
-    if (functionId < 0) {
-      status |= BLK_DATA_DATA_LOAD;
-      return status;
-    } else {
-      //      status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId);
-      //      if ((status & BLK_DATA_DATA_LOAD) == BLK_DATA_DATA_LOAD) {
-      //        return status;
-      //      }
-    }
-  }
-
-  return status;
-#endif
-  return 0;
-}
-
 int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
                               uint32_t* status) {
   *status = BLK_DATA_NOT_LOAD;
@@ -1802,7 +1775,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
   } else {
     taosMemoryFree(pMsg->pData);
     pSourceDataInfo->code = code;
-    qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code));
+    qDebug("%s fetch rsp received, index:%d, code:%s", pSourceDataInfo->taskId, index, tstrerror(code));
   }
 
   pSourceDataInfo->status = EX_SOURCE_DATA_READY;
@@ -1857,6 +1830,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
     SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq));
     if (NULL == pMsg) {
       pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+      taosMemoryFree(pWrapper);
       return pTaskInfo->code;
     }
 
@@ -2815,6 +2789,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
       blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId);
 
       blockDataCleanup(pInfo->pRes);
+      blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
+      blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
       doApplyScalarCalculation(pOperator, pBlock, order, scanFlag);
 
       if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) {
@@ -3396,8 +3372,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) {
     SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++];
     pSchema->colId = pColNode->colId;
     pSchema->type = pColNode->node.resType.type;
-    pSchema->type = pColNode->node.resType.bytes;
+    pSchema->bytes = pColNode->node.resType.bytes;
-    strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
+    tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
   }
 
   // this the tags and pseudo function columns, we only keep the tag columns
@@ -3411,7 +3387,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) {
       SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++];
       pSchema->colId = pColNode->colId;
       pSchema->type = pColNode->node.resType.type;
-      pSchema->type = pColNode->node.resType.bytes;
+      pSchema->bytes = pColNode->node.resType.bytes;
       strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
     }
   }

source/libs/executor/src/scanoperator.c

@@ -284,6 +284,7 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
   return true;
 }
 
+
 static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
   if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
     SExprSupp* pSup = &pTableScanInfo->pseudoSup;
@@ -324,19 +325,18 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
   } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
     qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
            pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->skipBlocks += 1;
-    doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
 
+    doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
+    pCost->skipBlocks += 1;
     return TSDB_CODE_SUCCESS;
   } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
     pCost->loadBlockStatis += 1;
     loadSMA = true;  // mark the operation of load sma;
     bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo);
     if (success) {  // failed to load the block sma data, data block statistics does not exist, load data block instead
-      doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
-
       qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
              pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+      doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
       return TSDB_CODE_SUCCESS;
     } else {
       qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo));
@@ -386,8 +386,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
   }
 
   relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true);
-
-  // currently only the tbname pseudo column
   doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
 
   if (pTableScanInfo->pFilterNode != NULL) {
@@ -1086,12 +1084,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
 
 static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
                                     SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) {
-  SResultRowInfo dumyInfo;
+  SResultRowInfo dumyInfo = {0};
   dumyInfo.cur.pageId = -1;
   STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
   STimeWindow endWin = win;
   STimeWindow preWin = win;
   uint64_t    groupId = gpIdCol[*pRowIndex];
+
   while (1) {
     if (hasGroup) {
       (*pRowIndex) += 1;
@@ -1155,6 +1154,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
           pResult->info.rows++;
         }
       }
+
+      blockDataDestroy(tmpBlock);
+
       if (pResult->info.rows > 0) {
         pResult->info.calWin = pInfo->updateWin;
         return pResult;
@@ -1327,6 +1329,11 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock*
   ASSERT(pResBlock->info.rows == 1);
 
   // build tagArray
+  /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/
+  /*STagVal tagVal = {*/
+  /*.cid = 0,*/
+  /*.type = 0,*/
+  /*};*/
   // build STag
   // set STag
 
@@ -1545,7 +1552,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
         tsdbReaderClose(pTSInfo->dataReader);
         pTSInfo->dataReader = NULL;
         tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
-        qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1);
+        qDebug("queue scan tsdb over, switch to wal ver %"PRId64, pTaskInfo->streamInfo.snapshotVer + 1);
         if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
           return NULL;
         }
@@ -2176,6 +2183,19 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     }
   }
 
+  if (pTableScanNode->pTags != NULL) {
+    int32_t    numOfTags;
+    SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags);
+    if (pTagExpr == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _error;
+    }
+    if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _error;
+    }
+  }
+
   pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES);
   if (pInfo->pBlockLists == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -2554,16 +2574,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
 
     SMetaReader smrChildTable = {0};
     metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
-    metaGetTableEntryByName(&smrChildTable, condTableName);
+    int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
+    if (code != TSDB_CODE_SUCCESS) {
+      // terrno has been set by metaGetTableEntryByName, therefore, return directly
+      return NULL;
+    }
+
     if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
       metaReaderClear(&smrChildTable);
       blockDataDestroy(dataBlock);
       pInfo->loadInfo.totalRows = 0;
       return NULL;
     }
+
     SMetaReader smrSuperTable = {0};
     metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK);
-    metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+    code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+    if (code != TSDB_CODE_SUCCESS) {
+      // terrno has been set by metaGetTableEntryByUid
+      return NULL;
+    }
+
     sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
     metaReaderClear(&smrSuperTable);
     metaReaderClear(&smrChildTable);
@@ -2970,7 +3001,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
 
     while (1) {
       int64_t startTs = taosGetTimestampUs();
-      strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
+      tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
       strcpy(pInfo->req.user, pInfo->pUser);
 
       int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req);
@@ -3781,6 +3812,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
 
   int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
   if (code != TSDB_CODE_SUCCESS) {
+    taosArrayDestroy(pColList);
     goto _error;
   }
 

source/libs/executor/src/sortoperator.c

@@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
 
   blockDataDestroy(p);
 
-  qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
+  qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
          pDataBlock->info.rows);
   return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
 }

source/libs/executor/src/tfill.c

@@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
                                                  &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
   pInfo->pCondition = pPhyFillNode->node.pConditions;
   pInfo->pColMatchColInfo = pColMatchColInfo;
-  initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+  int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
   pInfo->srcRowIndex = 0;
 
   pOperator->name = "FillOperator";
@@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
   pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo,
                                          NULL, NULL, NULL);
 
-  int32_t code = appendDownstream(pOperator, &downstream, 1);
+  code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
 _error:
   destroyStreamFillOperatorInfo(pInfo);
   taosMemoryFreeClear(pOperator);
+  pTaskInfo->code = code;
   return NULL;
 }

source/libs/executor/src/timewindowoperator.c

@@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
   SColumnInfoData*             pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   uint64_t*                    pGpDatas = (uint64_t*)pGpCol->pData;
   for (int32_t i = 0; i < pBlock->info.rows; i++) {
-    SResultRowInfo dumyInfo;
+    SResultRowInfo dumyInfo = {0};
     dumyInfo.cur.pageId = -1;
+
     STimeWindow win = {0};
     if (IS_FINAL_OP(pInfo)) {
       win.skey = startTsCols[i];
@@ -2841,14 +2842,13 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
   int32_t      numOfCols = 0;
   SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
   SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);
+  initBasicInfo(&pInfo->binfo, pResBlock);
 
   int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
-  initBasicInfo(&pInfo->binfo, pResBlock);
-
   pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
   pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
   pInfo->gap = pSessionNode->gap;
@@ -4665,7 +4665,6 @@ void destroyStreamStateOperatorInfo(void* param) {
       SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
       destroyStreamSessionAggOperatorInfo(pChInfo);
       taosMemoryFreeClear(pChild);
-      taosMemoryFreeClear(pChInfo);
     }
   }
   colDataDestroy(&pInfo->twAggSup.timeWindowData);
@@ -5828,27 +5827,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
   pInfo->isFinal = false;
 
-  if (pIntervalPhyNode->window.pExprs != NULL) {
+
-    int32_t    numOfScalar = 0;
-    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
-    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
-    if (code != TSDB_CODE_SUCCESS) {
-      goto _error;
-    }
-  }
 
   pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   SExprSupp* pSup = &pOperator->exprSupp;
+
+  initBasicInfo(&pInfo->binfo, pResBlock);
+  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
+  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
+
   size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
   int32_t    code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
-  initBasicInfo(&pInfo->binfo, pResBlock);
+  if (pIntervalPhyNode->window.pExprs != NULL) {
-  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
+    int32_t    numOfScalar = 0;
-  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
+    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
+    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+  }
 
   pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
   pInfo->invertible = false;

source/libs/executor/src/tsimplehash.c

@@ -18,7 +18,7 @@
 #include "tlog.h"
 
 #define SHASH_DEFAULT_LOAD_FACTOR 0.75
-#define HASH_MAX_CAPACITY         (1024 * 1024 * 16)
+#define HASH_MAX_CAPACITY         (1024 * 1024 * 16L)
 #define SHASH_NEED_RESIZE(_h)     ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
 
 #define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl))
@@ -104,7 +104,7 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) {
 
   int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
   if (newCapacity > HASH_MAX_CAPACITY) {
-    uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached",
+    uDebug("current capacity:%"PRIzu", maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached",
            pHashObj->capacity, HASH_MAX_CAPACITY);
     return;
   }

source/libs/function/src/builtinsimpl.c

@@ -6169,6 +6169,99 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+int32_t interpFunction(SqlFunctionCtx* pCtx) {
+#if 0
+  int32_t fillType = (int32_t) pCtx->param[2].i64;
+  //bool ascQuery = (pCtx->order == TSDB_ORDER_ASC);
+
+  if (pCtx->start.key == pCtx->startTs) {
+    assert(pCtx->start.key != INT64_MIN);
+
+    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->start.val);
+
+    goto interp_success_exit;
+  } else if (pCtx->end.key == pCtx->startTs && pCtx->end.key != INT64_MIN && fillType == TSDB_FILL_NEXT) {
+    COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->end.val);
+
+    goto interp_success_exit;
+  }
+
+  switch (fillType) {
+    case TSDB_FILL_NULL:
+      setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+      break;
+
+    case TSDB_FILL_SET_VALUE:
+      tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true);
+      break;
+
+    case TSDB_FILL_LINEAR:
+      if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs
+          || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
+        goto interp_exit;
+      }
+
+      double v1 = -1, v2 = -1;
+      GET_TYPED_DATA(v1, double, pCtx->inputType, &pCtx->start.val);
+      GET_TYPED_DATA(v2, double, pCtx->inputType, &pCtx->end.val);
+
+      SPoint point1 = {.key = pCtx->start.key, .val = &v1};
+      SPoint point2 = {.key = pCtx->end.key, .val = &v2};
+      SPoint point  = {.key = pCtx->startTs, .val = pCtx->pOutput};
+
+      int32_t srcType = pCtx->inputType;
+      if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) {
+        setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+      } else {
+        bool exceedMax = false, exceedMin = false;
+        taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, &exceedMax, &exceedMin);
+        if (exceedMax || exceedMin) {
+          __compar_fn_t func = getComparFunc((int32_t)pCtx->inputType, 0);
+          if (func(&pCtx->start.val, &pCtx->end.val) <= 0) {
+            COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->start.val : &pCtx->end.val);
+          } else {
+            COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->end.val : &pCtx->start.val);
+          }
+        }
+      }
+      break;
+
+    case TSDB_FILL_PREV:
+      if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs) {
+        goto interp_exit;
+      }
+
+      COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->start.val);
+      break;
+
+    case TSDB_FILL_NEXT:
+      if (pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
+        goto interp_exit;
+      }
+
+      COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->end.val);
+      break;
+
+    case TSDB_FILL_NONE:
+      // do nothing
+    default:
+      goto interp_exit;
+  }
+
+
+  interp_success_exit:
+  *(TSKEY*)pCtx->ptsOutputBuf = pCtx->startTs;
+  INC_INIT_VAL(pCtx, 1);
+
+  interp_exit:
+  pCtx->start.key = INT64_MIN;
+  pCtx->end.key = INT64_MIN;
+  pCtx->endTs = pCtx->startTs;
+#endif
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) {
   int32_t numOfElems = 0;
 

source/libs/parser/src/parTranslater.c

@@ -2357,7 +2357,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo
 }
 
 static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSelect) {
-  if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) {
+  if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) ||
+      TSDB_SYSTEM_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType) {
     return TSDB_CODE_SUCCESS;
   }
 

source/libs/sync/inc/syncInt.h

@@ -169,7 +169,7 @@ typedef struct SSyncNode {
 } SSyncNode;
 
 // open/close --------------
-SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
+SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo);
 void       syncNodeStart(SSyncNode* pSyncNode);
 void       syncNodeStartStandBy(SSyncNode* pSyncNode);
 void       syncNodeClose(SSyncNode* pSyncNode);

source/libs/sync/src/syncAppendEntries.c

@@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
             SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset);
             code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
             if (code != 0) {
+              sError("vgId:%d, failed to append log entry since %s",  ths->vgId, tstrerror(terrno));
               return -1;
             }
 
@@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
             SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset);
             code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
             if (code != 0) {
+              sError("vgId:%d, failed to append log entry since %s",  ths->vgId, tstrerror(terrno));
               return -1;
             }
 
@@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
 
         code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
         if (code != 0) {
+          sError("vgId:%d, failed to append log entry since %s",  ths->vgId, tstrerror(terrno));
           return -1;
         }
 
@@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
 
         code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
         if (code != 0) {
+          sError("vgId:%d, failed to append log entry since %s",  ths->vgId, tstrerror(terrno));
           return -1;
         }
 

source/libs/sync/src/syncCommit.c

@@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
         pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
       } else {
         pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index);
-        ASSERT(pEntry != NULL);
+        if (pEntry == NULL) {
+           sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index);
+           return;
+        }
       }
       // cannot commit, even if quorum agree. need check term!
       if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) {
@@ -127,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
     // execute fsm
     if (pSyncNode->pFsm != NULL) {
       int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state);
-      ASSERT(code == 0);
+      if (code != 0) {
+         wError("failed to commit sync node since %s", tstrerror(terrno));
+      }
     }
   }
 }

source/libs/sync/src/syncMain.c

@@ -86,10 +86,10 @@ void syncCleanUp() {
   }
 }
 
-int64_t syncOpen(const SSyncInfo* pSyncInfo) {
+int64_t syncOpen(SSyncInfo* pSyncInfo) {
   SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
   if (pSyncNode == NULL) {
-    sError("failed to open sync node. vgId:%d", pSyncInfo->vgId);
+    sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr());
     return -1;
   }
 
@@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg
   return ret;
 }
 
-int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
+int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) {
   SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
   if (pSyncNode == NULL) {
     terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
@@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
   }
   ASSERT(rid == pSyncNode->rid);
 
+#if 0
   if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) {
     taosReleaseRef(tsNodeRefId, pSyncNode->rid);
     terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR;
@@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
 
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);
   return ret;
+#else
+  syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg);
+  syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID);
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+  return 0;
+#endif
 }
 
 int32_t syncLeaderTransfer(int64_t rid) {
@@ -919,9 +926,7 @@ _END:
 }
 
 // open/close --------------
-SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
+SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
-  SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo;
-
   SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode));
   if (pSyncNode == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
       sError("failed to open raft cfg file. path:%s", pSyncNode->configPath);
       goto _error;
     }
-    pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
+    if (pSyncInfo->syncCfg.replicaNum == 0) {
+      pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
+    }
 
     raftCfgClose(pSyncNode->pRaftCfg);
     pSyncNode->pRaftCfg = NULL;
@@ -2650,7 +2657,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
     int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
-    ASSERT(code == 0);
+    if (code != 0) {
+      sError("vgId:%d, failed to append log entry since %s",  ths->vgId, tstrerror(terrno));
+      return -1;
+    }
     syncNodeReplicate(ths, false);
   }
 
@@ -2726,7 +2736,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
     code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
     if (code != 0) {
       // del resp mgr, call FpCommitCb
-      ASSERT(0);
+      sError("vgId:%d, failed to append log entry since %s",  ths->vgId, tstrerror(terrno));
       return -1;
     }
 
@@ -2790,8 +2800,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p
 
     code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
     if (code != 0) {
+      sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
       // del resp mgr, call FpCommitCb
-      ASSERT(0);
       return -1;
     }
 
@@ -3043,7 +3053,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex,
           pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
         } else {
           code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry);
-          ASSERT(code == 0);
+          if (code != 0) {
+              sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i);
+              return -1;
+           }
           ASSERT(pEntry != NULL);
         }
 

source/libs/sync/src/syncRaftLog.c

@@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
     snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
              pEntry->index, err, err, errStr, sysErr, sysErrStr);
     syncNodeErrorLog(pData->pSyncNode, logBuf);
-
-    ASSERT(0);
     return -1;
   }
   pEntry->index = index;
@@ -414,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   return 0;
 }
 
-SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
+SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
 
   if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
-    taosThreadMutexLock(&(pData->mutex));
-
     // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
     SWalReader* pWalHandle = pData->pWalHandle;
     ASSERT(pWalHandle != NULL);
@@ -444,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
         }
       } while (0);
 
-      ASSERT(0);
+      sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index);
+      return NULL;
     }
 
     SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
@@ -465,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
         terrno = saveErr;
     */
 
-    taosThreadMutexUnlock(&(pData->mutex));
     return pEntry;
 
   } else {
@@ -473,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
   }
 }
 
+SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
+    SSyncLogStoreData* pData = pLogStore->data;
+    SSyncRaftEntry *pEntry = NULL;
+
+    taosThreadMutexLock(&pData->mutex);
+    pEntry = logStoreGetEntryWithoutLock(pLogStore, index);
+    taosThreadMutexUnlock(&pData->mutex);
+    return pEntry;
+}
+
 int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;

source/libs/tdb/src/db/tdbBtree.c

@@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
 
     ret = tdbPagerWrite(pPager, pPage);
     if (ret < 0) {
+      tdbError("failed to write page since %s", terrstr());
       return -1;
     }
 
@@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN
 
   ret = tdbPagerWrite(pPager, pChild);
   if (ret < 0) {
-    // TODO
+    tdbError("failed to write page since %s", terrstr());
-    ASSERT(0);
+    return -1;
-    return 0;
   }
 
   // Copy the root page content to the child page
@@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
 
       ret = tdbPagerWrite(pBt->pPager, pOlds[i]);
       if (ret < 0) {
-        // TODO
+        tdbError("failed to write page since %s", terrstr());
-        ASSERT(0);
         return -1;
       }
     }
@@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
 
     ret = tdbPagerWrite(pBt->pPager, pParent);
     if (ret < 0) {
-      // TODO
+      tdbError("failed to write page since %s", terrstr());
-      ASSERT(0);
       return -1;
     }
 
@@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
 
         ret = tdbPagerWrite(pBt->pPager, pNews[iNew]);
         if (ret < 0) {
-          // TODO
+          tdbError("failed to write page since %s", terrstr());
-          ASSERT(0);
           return -1;
         }
       }
@@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt)
   // mark dirty
   ret = tdbPagerWrite(pBt->pPager, *ppOfp);
   if (ret < 0) {
-    ASSERT(0);
+    tdbError("failed to write page since %s", terrstr());
     return -1;
   }
 
@@ -987,7 +984,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
 
     // fetch a new ofp and make it dirty
     SPgno  pgno = 0;
-    SPage *ofp, *nextOfp;
+    SPage *ofp = NULL, *nextOfp = NULL;
 
     ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt);
     if (ret < 0) {
@@ -1009,6 +1006,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
       nLeft -= kLen;
       // pack partial val to local if any space left
       if (nLocal > nHeader + kLen + sizeof(SPgno)) {
+        ASSERT(pVal != NULL && vLen != 0);
         memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno));
         nLeft -= nLocal - nHeader - kLen - sizeof(SPgno);
       }
@@ -1942,7 +1940,7 @@ int tdbBtcDelete(SBTC *pBtc) {
   // drop the cell on the leaf
   ret = tdbPagerWrite(pPager, pBtc->pPage);
   if (ret < 0) {
-    ASSERT(0);
+    tdbError("failed to write page since %s", terrstr());
     return -1;
   }
 
@@ -1964,7 +1962,7 @@ int tdbBtcDelete(SBTC *pBtc) {
         if (idx < nCells) {
           ret = tdbPagerWrite(pPager, pPage);
           if (ret < 0) {
-            ASSERT(0);
+            tdbError("failed to write page since %s", terrstr());
             return -1;
           }
 
@@ -2029,7 +2027,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int
   // mark dirty
   ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage);
   if (ret < 0) {
-    ASSERT(0);
+    tdbError("failed to write page since %s", terrstr());
     return -1;
   }
 

source/libs/tdb/src/db/tdbDb.c

@@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) {
   for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
     ret = tdbPagerBegin(pPager, pTxn);
     if (ret < 0) {
-      ASSERT(0);
+      tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
       return -1;
     }
   }
@@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) {
   for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
     ret = tdbPagerCommit(pPager, pTxn);
     if (ret < 0) {
-      ASSERT(0);
+      tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
       return -1;
     }
   }
@@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) {
   for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
     ret = tdbPagerAbort(pPager, pTxn);
     if (ret < 0) {
-      ASSERT(0);
+      tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
       return -1;
     }
   }

source/libs/tdb/src/db/tdbPCache.c

@@ -105,6 +105,7 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
     for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) {
       if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) {
         // TODO: handle error
+        tdbOsFree(pCache->aPage);
         return -1;
       }
 
@@ -267,7 +268,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
   // 4. Try a create new page
   if (!pPage) {
     ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg);
-    if (ret < 0) {
+    if (ret < 0 && pPage != NULL) {
       // TODO
       ASSERT(0);
       return NULL;
@@ -300,8 +301,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
       pPage->pPager = pPageH->pPager;
 
       memcpy(pPage->pData, pPageH->pData, pPage->pageSize);
-      tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage,
+      // tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage,
-               TDB_PAGE_PGNO(pPageH));
+      // TDB_PAGE_PGNO(pPageH));
       tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize);
       pPage->kLen = pPageH->kLen;
       pPage->vLen = pPageH->vLen;

source/libs/tdb/src/db/tdbPager.c

@@ -84,7 +84,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
   pPager->pCache = pCache;
 
   pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
-  if (pPager->fd < 0) {
+  if (TDB_FD_INVALID(pPager->fd)) {
+    // if (pPager->fd < 0) {
     return -1;
   }
 
@@ -156,6 +157,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) {
 
     ret = tdbPagerWrite(pPager, pPage);
     if (ret < 0) {
+      tdbError("failed to write page since %s", terrstr());
       return -1;
     }
 
@@ -210,7 +212,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
   if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
     ret = tdbPagerWritePageToJournal(pPager, pPage);
     if (ret < 0) {
-      ASSERT(0);
+      tdbError("failed to write page to journal since %s", tstrerror(terrno));
       return -1;
     }
   }
@@ -225,7 +227,9 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
 
   // Open the journal
   pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
-  if (pPager->jfd < 0) {
+  if (TDB_FD_INVALID(pPager->jfd)) {
+    tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
@@ -243,9 +247,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
   // sync the journal file
   ret = tdbOsFSync(pPager->jfd);
   if (ret < 0) {
-    // TODO
+    tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
-    ASSERT(0);
+    terrno = TAOS_SYSTEM_ERROR(errno);
-    return 0;
+    return -1;
   }
 
   // loop to write the dirty pages to file
@@ -255,7 +259,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
     pPage = (SPage *)pNode;
     ret = tdbPagerWritePageToDB(pPager, pPage);
     if (ret < 0) {
-      ASSERT(0);
+      tdbError("failed to write page to db since %s", tstrerror(terrno));
       return -1;
     }
   }
@@ -277,11 +281,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
   tRBTreeCreate(&pPager->rbt, pageCmpFn);
 
   // sync the db file
-  tdbOsFSync(pPager->fd);
+  if (tdbOsFSync(pPager->fd) < 0) {
+    tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
 
   // remove the journal file
-  tdbOsClose(pPager->jfd);
+  if (tdbOsClose(pPager->jfd) < 0) {
-  tdbOsRemove(pPager->jFileName);
+    tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+    tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
   pPager->inTran = 0;
 
   return 0;
@@ -297,14 +315,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
   // 0, sync the journal file
   ret = tdbOsFSync(pPager->jfd);
   if (ret < 0) {
-    // TODO
+    tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
-    ASSERT(0);
+    terrno = TAOS_SYSTEM_ERROR(errno);
-    return 0;
+    return -1;
   }
 
   tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755);
   if (jfd == NULL) {
-    return 0;
+    return -1;
   }
 
   ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
@@ -348,7 +366,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
 
   // 4, remove the journal file
   tdbOsClose(pPager->jfd);
-  tdbOsRemove(pPager->jFileName);
+  (void)tdbOsRemove(pPager->jFileName);
   pPager->inTran = 0;
 
   return 0;
@@ -516,11 +534,16 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
 
   ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno));
   if (ret < 0) {
+    tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
   ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize);
   if (ret < 0) {
+    tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName,
+             pPage->pageSize);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
@@ -540,13 +563,16 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
 
   offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
   if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
-    ASSERT(0);
+    tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
   ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize);
   if (ret < 0) {
-    ASSERT(0);
+    tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName,
+             pPage->pageSize);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
@@ -580,33 +606,54 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
 
     int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
     if (ret < 0) {
+      tdbOsFree(pageBuf);
       return -1;
     }
 
     ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
     if (ret < 0) {
+      tdbOsFree(pageBuf);
       return -1;
     }
 
     i64 offset = pPager->pageSize * (pgno - 1);
     if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
-      ASSERT(0);
+      tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tdbOsFree(pageBuf);
       return -1;
     }
 
     ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize);
     if (ret < 0) {
-      ASSERT(0);
+      tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
+               pPager->pageSize);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tdbOsFree(pageBuf);
       return -1;
     }
   }
 
-  tdbOsFSync(pPager->fd);
+  if (tdbOsFSync(pPager->fd) < 0) {
+    tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tdbOsFree(pageBuf);
+    return -1;
+  }
 
   tdbOsFree(pageBuf);
 
-  tdbOsClose(jfd);
+  if (tdbOsClose(jfd) < 0) {
-  tdbOsRemove(pPager->jFileName);
+    tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+    tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
 
   return 0;
 }

source/libs/tdb/src/db/tdbTable.c

@@ -106,11 +106,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
   // pTb->pBt
   ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt));
   if (ret < 0) {
+    tdbOsFree(pTb);
     return -1;
   }
 
   ret = tdbPagerRestore(pPager, pTb->pBt);
   if (ret < 0) {
+    tdbOsFree(pTb);
     return -1;
   }
 

source/libs/tdb/src/inc/tdbOs.h

@@ -37,6 +37,8 @@ extern "C" {
 /* file */
 typedef TdFilePtr tdb_fd_t;
 
+#define TDB_FD_INVALID(fd) (fd == NULL)
+
 #define TDB_O_CREAT  TD_FILE_CREATE
 #define TDB_O_WRITE  TD_FILE_WRITE
 #define TDB_O_READ   TD_FILE_READ

source/libs/wal/src/walMeta.c

@@ -35,113 +35,251 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe
 
 int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; }
 
-static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
+static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
-  sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
+  return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
 }
 
-static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
+static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) {
-  int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+  return sprintf(buf, "%s/meta-ver.tmp", pWal->path);
-  ASSERT(sz > 0);
+}
-#if 0
-  for (int i = 0; i < sz; i++) {
-    SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
-  }
-#endif
 
-  SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1);
+static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
+  int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+  terrno = TSDB_CODE_SUCCESS;
+  ASSERT(fileIdx >= 0 && fileIdx < sz);
+
+  SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
   char          fnameStr[WAL_FILE_LEN];
-  walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr);
+  walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
 
   int64_t fileSize = 0;
   taosStatFile(fnameStr, &fileSize, NULL);
-  int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize);
-  pLastFileInfo->fileSize = fileSize;
 
-  TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
+  TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
   if (pFile == NULL) {
+    wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
+  // ensure size as non-negative
+  pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize);
+
   uint64_t magic = WAL_MAGIC;
+  int64_t  walCkHeadSz = sizeof(SWalCkHead);
+  int64_t  end = fileSize;
+  int64_t  offset = 0;
+  int32_t  capacity = 0;
+  int32_t  readSize = 0;
+  char*    buf = NULL;
+  char*    found = NULL;
+  bool     firstTrial = pFileInfo->fileSize < fileSize;
 
-  char* buf = taosMemoryMalloc(readSize + 5);
+  // search for the valid last WAL entry, e.g. block by block
-  if (buf == NULL) {
-    taosCloseFile(&pFile);
-    terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  int64_t offset;
-  offset = taosLSeekFile(pFile, -readSize, SEEK_END);
-  if (readSize != taosReadFile(pFile, buf, readSize)) {
-    taosMemoryFree(buf);
-    taosCloseFile(&pFile);
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  char* found = NULL;
   while (1) {
+    offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE);
+    ASSERT(offset <= end);
+    readSize = end - offset;
+    capacity = readSize + sizeof(magic);
+
+    int64_t limit = WAL_RECOV_SIZE_LIMIT;
+    if (limit < readSize) {
+      wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64
+             ", end:%" PRId64 ", file:%s",
+             pWal->cfg.vgId, limit, offset, end, fnameStr);
+      terrno = TSDB_CODE_WAL_SIZE_LIMIT;
+      goto _err;
+    }
+
+    void* ptr = taosMemoryRealloc(buf, capacity);
+    if (ptr == NULL) {
+      terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
+      goto _err;
+    }
+    buf = ptr;
+
+    int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET);
+    if (ret < 0) {
+      wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+
+    if (readSize != taosReadFile(pFile, buf, readSize)) {
+      wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+             readSize, fnameStr);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+
+    char* candidate = NULL;
     char* haystack = buf;
-    char* candidate;
+
-    while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) {
+    while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) {
-      // read and validate
+      // validate head
-      SWalCkHead* logContent = (SWalCkHead*)candidate;
+      int64_t len = readSize - (candidate - buf);
-      if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) {
+      if (len < walCkHeadSz) {
-        found = candidate;
+        break;
       }
+      SWalCkHead* logContent = (SWalCkHead*)candidate;
+      if (walValidHeadCksum(logContent) != 0) {
+        wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s",
+               ((char*)(logContent)-buf), fnameStr);
+        haystack = candidate + 1;
+        if (firstTrial) {
+          break;
+        } else {
+          continue;
+        }
+      }
+
+      // validate body
+      int64_t size = walCkHeadSz + logContent->head.bodyLen;
+      if (len < size) {
+        int64_t extraSize = size - len;
+        if (capacity < readSize + extraSize + sizeof(magic)) {
+          capacity += extraSize;
+          void* ptr = taosMemoryRealloc(buf, capacity);
+          if (ptr == NULL) {
+            terrno = TSDB_CODE_OUT_OF_MEMORY;
+            goto _err;
+          }
+          buf = ptr;
+        }
+        int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET);
+        if (ret < 0) {
+          wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno),
+                 offset);
+          terrno = TAOS_SYSTEM_ERROR(errno);
+          break;
+        }
+        if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) {
+          wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s",
+                 pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr);
+          terrno = TAOS_SYSTEM_ERROR(errno);
+          break;
+        }
+      }
+      if (walValidBodyCksum(logContent) != 0) {
+        terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
+        wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s",
+               ((char*)(logContent)-buf), fnameStr);
+        haystack = candidate + 1;
+        if (firstTrial) {
+          break;
+        } else {
+          continue;
+        }
+      }
+
+      // found one
+      found = candidate;
       haystack = candidate + 1;
     }
+
     if (found || offset == 0) break;
-    offset = TMIN(0, offset - readSize + sizeof(uint64_t));
+
-    int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET);
+    // go backwards, e.g. by at most one WAL scan buf size
-    ASSERT(offset == offset2);
+    end = offset + walCkHeadSz - 1;
-    if (readSize != taosReadFile(pFile, buf, readSize)) {
+    firstTrial = false;
-      taosMemoryFree(buf);
-      taosCloseFile(&pFile);
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
-    }
-#if 0
-    if (found == buf) {
-      SWalCkHead* logContent = (SWalCkHead*)found;
-      if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) {
-        // file has to be deleted
-        taosMemoryFree(buf);
-        taosCloseFile(&pFile);
-        terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
-        return -1;
-      }
-    }
-#endif
   }
 
-  if (found == NULL) {
+  // determine end of last entry
-    // file corrupted, no complete log
+  SWalCkHead* lastEntry = (SWalCkHead*)found;
-    // TODO delete and search in previous files
+  int64_t     retVer = -1;
-    /*ASSERT(0);*/
+  int64_t     lastEntryBeginOffset = 0;
-    terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+  int64_t     lastEntryEndOffset = 0;
-    return -1;
+
+  if (lastEntry == NULL) {
+    terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+  } else {
+    retVer = lastEntry->head.version;
+    lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf);
+    lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen;
   }
 
   // truncate file
-  SWalCkHead* lastEntry = (SWalCkHead*)found;
-  int64_t     retVer = lastEntry->head.version;
-  int64_t     lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf);
-  int64_t     lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen;
   if (lastEntryEndOffset != fileSize) {
-    wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
+    wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
           fileSize);
-    taosFtruncateFile(pFile, lastEntryEndOffset);
+    if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) {
-    ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset;
+      wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr);
-    pWal->totSize -= (fileSize - lastEntryEndOffset);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+    if (taosFsyncFile(pFile) < 0) {
+      wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
   }
+  pFileInfo->fileSize = lastEntryEndOffset;
 
   taosCloseFile(&pFile);
   taosMemoryFree(buf);
-
   return retVer;
+
+_err:
+  taosCloseFile(&pFile);
+  taosMemoryFree(buf);
+  return -1;
+}
+
+static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
+  int metaFileNum = taosArrayGetSize(metaLogList);
+  int actualFileNum = taosArrayGetSize(actualLogList);
+  int j = 0;
+
+  // both of the lists in asc order
+  for (int i = 0; i < actualFileNum; i++) {
+    SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i);
+    while (j < metaFileNum) {
+      SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j);
+      ASSERT(pMetaInfo != NULL);
+      if (pMetaInfo->firstVer < pLogInfo->firstVer) {
+        j++;
+      } else if (pMetaInfo->firstVer == pLogInfo->firstVer) {
+        (*pLogInfo) = *pMetaInfo;
+        j++;
+        break;
+      } else {
+        break;
+      }
+    }
+  }
+
+  taosArrayClear(metaLogList);
+
+  for (int i = 0; i < actualFileNum; i++) {
+    SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i);
+    taosArrayPush(metaLogList, pFileInfo);
+  }
+}
+
+void walAlignVersions(SWal* pWal) {
+  if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) {
+    wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId,
+          pWal->vers.firstVer, pWal->vers.snapshotVer);
+    pWal->vers.firstVer = pWal->vers.snapshotVer + 1;
+  }
+  if (pWal->vers.lastVer < pWal->vers.snapshotVer) {
+    wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+          pWal->vers.lastVer, pWal->vers.snapshotVer);
+    pWal->vers.lastVer = pWal->vers.snapshotVer;
+  }
+  if (pWal->vers.commitVer < pWal->vers.snapshotVer) {
+    wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+          pWal->vers.commitVer, pWal->vers.snapshotVer);
+    pWal->vers.commitVer = pWal->vers.snapshotVer;
+  }
+  if (pWal->vers.appliedVer < pWal->vers.snapshotVer) {
+    wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+          pWal->vers.appliedVer, pWal->vers.snapshotVer);
+    pWal->vers.appliedVer = pWal->vers.snapshotVer;
+  }
+
+  pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer);
+  pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer);
 }
 
 int walCheckAndRepairMeta(SWal* pWal) {
@@ -150,7 +288,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
   const char* idxPattern = "^[0-9]+.idx$";
   regex_t     logRegPattern;
   regex_t     idxRegPattern;
-  bool        fixed = false;
 
   regcomp(&logRegPattern, logPattern, REG_EXTENDED);
   regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
@@ -184,225 +321,237 @@ int walCheckAndRepairMeta(SWal* pWal) {
 
   taosArraySort(actualLog, compareWalFileInfo);
 
-  int metaFileNum = taosArrayGetSize(pWal->fileInfoSet);
+  int     metaFileNum = taosArrayGetSize(pWal->fileInfoSet);
-  int actualFileNum = taosArrayGetSize(actualLog);
+  int     actualFileNum = taosArrayGetSize(actualLog);
+  int64_t firstVerPrev = pWal->vers.firstVer;
+  int64_t lastVerPrev = pWal->vers.lastVer;
+  int64_t totSize = 0;
+  bool    updateMeta = (metaFileNum != actualFileNum);
 
-#if 0
+  // rebuild meta of file info
-  for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) {
+  walRebuildFileInfoSet(pWal->fileInfoSet, actualLog);
-    SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo);
+  taosArrayDestroy(actualLog);
+
+  int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+  ASSERT(sz == actualFileNum);
+
+  // scan and determine the lastVer
+  int32_t fileIdx = sz;
+
+  while (--fileIdx >= 0) {
     char          fnameStr[WAL_FILE_LEN];
+    int64_t       fileSize = 0;
+    SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
+
     walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
-    int64_t fileSize = 0;
+    int32_t code = taosStatFile(fnameStr, &fileSize, NULL);
-    taosStatFile(fnameStr, &fileSize, NULL);
+    if (code < 0) {
-    if (fileSize == 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
+      return -1;
+    }
+
+    ASSERT(pFileInfo->firstVer >= 0);
+
+    if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) {
+      totSize += pFileInfo->fileSize;
+      continue;
+    }
+    updateMeta = true;
+
+    int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx);
+    if (lastVer < 0) {
+      if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) {
+        wError("failed to scan wal last ver since %s", terrstr());
+        return -1;
+      }
+      ASSERT(pFileInfo->fileSize == 0);
+      // remove the empty wal log, and its idx
       taosRemoveFile(fnameStr);
       walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
       taosRemoveFile(fnameStr);
-      taosArrayPop(pLogInfoArray);
+      // remove its meta entry
-    } else {
+      taosArrayRemove(pWal->fileInfoSet, fileIdx);
-      break;
+      continue;
     }
+
+    // update lastVer
+    pFileInfo->lastVer = lastVer;
+    totSize += pFileInfo->fileSize;
   }
 
-  actualFileNum = taosArrayGetSize(pLogInfoArray);
+  // reset vers info and so on
-#endif
-
-  {
-    int32_t i = 0, j = 0;
-    while (i < actualFileNum && j < metaFileNum) {
-      SWalFileInfo* pActualFile = taosArrayGet(actualLog, i);
-      SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j);
-      if (pActualFile->firstVer < pMetaFile->firstVer) {
-        char fNameStr[WAL_FILE_LEN];
-        walBuildLogName(pWal, pActualFile->firstVer, fNameStr);
-        taosRemoveFile(fNameStr);
-        walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
-        taosRemoveFile(fNameStr);
-        i++;
-      } else if (pActualFile->firstVer > pMetaFile->firstVer) {
-        taosArrayRemove(pWal->fileInfoSet, j);
-        metaFileNum--;
-      } else {
-        i++;
-        j++;
-      }
-    }
-    if (i == actualFileNum && j == metaFileNum) {
-      if (j > 0) {
-        SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1);
-        int64_t       fsize = 0;
-        char          fNameStr[WAL_FILE_LEN];
-        walBuildLogName(pWal, pLastInfo->firstVer, fNameStr);
-        taosStatFile(fNameStr, &fsize, NULL);
-        if (pLastInfo->fileSize != fsize) {
-          fixed = true;
-          pLastInfo->fileSize = fsize;
-          pLastInfo->lastVer = walScanLogGetLastVer(pWal);
-        }
-      }
-    } else {
-      fixed = true;
-      while (i < actualFileNum) {
-        SWalFileInfo* pActualFile = taosArrayGet(actualLog, i);
-        char          fNameStr[WAL_FILE_LEN];
-        walBuildLogName(pWal, pActualFile->firstVer, fNameStr);
-        taosStatFile(fNameStr, &pActualFile->fileSize, NULL);
-
-        if (pActualFile->fileSize == 0) {
-          ASSERT(i == actualFileNum - 1);
-          taosRemoveFile(fNameStr);
-
-          walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
-          taosRemoveFile(fNameStr);
-          break;
-        }
-
-        if (i < actualFileNum - 1) {
-          pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1;
-          taosArrayPush(pWal->fileInfoSet, pActualFile);
-          i++;
-        } else {
-          pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile);
-          pActualFile->lastVer = walScanLogGetLastVer(pWal);
-          if (pActualFile->lastVer == -1) {
-            taosRemoveFile(fNameStr);
-
-            walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
-            taosRemoveFile(fNameStr);
-            taosArrayPop(pWal->fileInfoSet);
-          }
-          break;
-        }
-      }
-    }
-  }
-
-#if 0
-  if (metaFileNum > actualFileNum) {
-    taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum);
-  } else if (metaFileNum < actualFileNum) {
-    for (int i = metaFileNum; i < actualFileNum; i++) {
-      SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i);
-      taosArrayPush(pWal->fileInfoSet, pFileInfo);
-    }
-  }
-#endif
-
-  taosArrayDestroy(actualLog);
-
   actualFileNum = taosArrayGetSize(pWal->fileInfoSet);
   pWal->writeCur = actualFileNum - 1;
-
+  pWal->totSize = totSize;
+  pWal->vers.lastVer = -1;
   if (actualFileNum > 0) {
-    int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer;
+    pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
-    if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) {
+    pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer;
-      fixed = true;
+  }
-      pWal->vers.lastVer = fLastVer;
+  (void)walAlignVersions(pWal);
-    }
+
-    int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
+  // update meta file
-    if (fFirstVer != pWal->vers.firstVer) {
+  if (updateMeta) {
-      fixed = true;
+    (void)walSaveMeta(pWal);
-      pWal->vers.firstVer = fFirstVer;
+  }
-    }
+  return 0;
+}
+
+int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
+  if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   }
 
-  if (fixed) {
+  if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
-    walSaveMeta(pWal);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (walValidHeadCksum(pCkHead) != 0) {
+    terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
+    return -1;
   }
 
   return 0;
 }
 
-int walCheckAndRepairIdx(SWal* pWal) {
+int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
   int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
-  for (int32_t i = 0; i < sz; i++) {
+  ASSERT(fileIdx >= 0 && fileIdx < sz);
-    SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
+  SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
+  char          fnameStr[WAL_FILE_LEN];
+  walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
+  char fLogNameStr[WAL_FILE_LEN];
+  walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
+  int64_t fileSize = 0;
 
-    char fnameStr[WAL_FILE_LEN];
+  if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) {
-    walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
+    wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
-    int64_t   fsize;
+    terrno = TAOS_SYSTEM_ERROR(errno);
-    TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
+    return -1;
-    if (pIdxFile == NULL) {
+  }
-      ASSERT(0);
+
+  ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer);
+  if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
+    return 0;
+  }
+
+  // start to repair
+  int64_t      offset = fileSize - fileSize % sizeof(SWalIdxEntry);
+  TdFilePtr    pLogFile = NULL;
+  TdFilePtr    pIdxFile = NULL;
+  SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)};
+  SWalCkHead   ckHead;
+  memset(&ckHead, 0, sizeof(ckHead));
+  ckHead.head.version = idxEntry.ver;
+
+  pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
+  if (pIdxFile == NULL) {
+    wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
+  if (pLogFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
+    goto _err;
+  }
+
+  // determine the last valid entry end, i.e. offset
+  while ((offset -= sizeof(SWalIdxEntry)) >= 0) {
+    if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
+      wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+             offset, fnameStr);
       terrno = TAOS_SYSTEM_ERROR(errno);
-      wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr());
+      goto _err;
-      return -1;
     }
 
-    taosFStatFile(pIdxFile, &fsize, NULL);
+    if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
-    if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
+      wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
-      taosCloseFile(&pIdxFile);
+             offset, fnameStr);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+
+    if (idxEntry.ver > pFileInfo->lastVer) {
       continue;
     }
 
-    int32_t left = fsize % sizeof(SWalIdxEntry);
+    if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
-    int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END);
+      wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "",
-    if (left != 0) {
+            pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver);
-      taosFtruncateFile(pIdxFile, offset);
+      continue;
-      wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr,
-            offset, fsize);
-    }
-    offset -= sizeof(SWalIdxEntry);
-
-    SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer};
-    while (1) {
-      if (offset < 0) {
-        taosLSeekFile(pIdxFile, 0, SEEK_SET);
-        taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
-        break;
-      }
-      if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        wError("vgId:%d cannot seek offset %ld when repair idx since %s", pWal->cfg.vgId, offset, terrstr());
-      }
-      int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
-      if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) {
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        return -1;
-      }
-      if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) {
-        taosFtruncateFile(pIdxFile, offset);
-        wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld",
-              pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset);
-        offset -= sizeof(SWalIdxEntry);
-      } else {
-        break;
-      }
     }
 
-    if (idxEntry.ver < pFileInfo->lastVer) {
+    if (idxEntry.ver == ckHead.head.version) {
-      char fLogNameStr[WAL_FILE_LEN];
+      break;
-      walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
+    }
-      TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
+  }
-      if (pLogFile == NULL) {
+  offset += sizeof(SWalIdxEntry);
-        terrno = TAOS_SYSTEM_ERROR(errno);
+
-        wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
+  // ftruncate idx file
-        return -1;
+  if (offset < fileSize) {
-      }
+    if (taosFtruncateFile(pIdxFile, offset) < 0) {
-      while (idxEntry.ver < pFileInfo->lastVer) {
+      wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
-        if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) {
+             strerror(errno), offset, fnameStr);
-          terrno = TAOS_SYSTEM_ERROR(errno);
+      terrno = TAOS_SYSTEM_ERROR(errno);
-          wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset,
+      goto _err;
-                 terrstr());
+    }
-          return -1;
+  }
-        }
+
-        SWalCkHead ckHead;
+  // rebuild idx file
-        taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead));
+  if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) {
-        if (idxEntry.ver != ckHead.head.version) {
+    wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
-          // todo truncate this idx also
+           offset, fnameStr);
-          taosCloseFile(&pLogFile);
+    terrno = TAOS_SYSTEM_ERROR(errno);
-          wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId,
+    goto _err;
-                 idxEntry.offset, idxEntry.ver, ckHead.head.version);
+  }
-          return -1;
+
-        }
+  while (idxEntry.ver < pFileInfo->lastVer) {
-        idxEntry.ver = ckHead.head.version + 1;
+    ASSERT(idxEntry.ver == ckHead.head.version);
-        idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen;
+
-        wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset);
+    idxEntry.ver += 1;
-        taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
+    idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
-      }
+
-      taosCloseFile(&pLogFile);
+    if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
+      wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
+             idxEntry.offset, fLogNameStr);
+      goto _err;
+    }
+    wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset);
+    if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) {
+      wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
+      goto _err;
+    }
+  }
+
+  if (taosFsyncFile(pIdxFile) < 0) {
+    wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
+    goto _err;
+  }
+
+  (void)taosCloseFile(&pLogFile);
+  (void)taosCloseFile(&pIdxFile);
+  return 0;
+
+_err:
+  (void)taosCloseFile(&pLogFile);
+  (void)taosCloseFile(&pIdxFile);
+  return -1;
+}
+
+int walCheckAndRepairIdx(SWal* pWal) {
+  int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+  int32_t fileIdx = sz;
+  while (--fileIdx >= 0) {
+    if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) {
+      wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx);
+      return -1;
     }
-    taosCloseFile(&pIdxFile);
   }
   return 0;
 }
@@ -495,14 +644,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
   ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0);
   cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField;
   pRoot = cJSON_Parse(bytes);
+  if (!pRoot) goto _err;
   pMeta = cJSON_GetObjectItem(pRoot, "meta");
+  if (!pMeta) goto _err;
   pField = cJSON_GetObjectItem(pMeta, "firstVer");
+  if (!pField) goto _err;
   pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField));
   pField = cJSON_GetObjectItem(pMeta, "snapshotVer");
+  if (!pField) goto _err;
   pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField));
   pField = cJSON_GetObjectItem(pMeta, "commitVer");
+  if (!pField) goto _err;
   pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField));
   pField = cJSON_GetObjectItem(pMeta, "lastVer");
+  if (!pField) goto _err;
   pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField));
 
   pFiles = cJSON_GetObjectItem(pRoot, "files");
@@ -512,17 +667,23 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
   taosArrayEnsureCap(pArray, sz);
   SWalFileInfo* pData = pArray->pData;
   for (int i = 0; i < sz; i++) {
-    cJSON*        pInfoJson = cJSON_GetArrayItem(pFiles, i);
+    cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
+    if (!pInfoJson) goto _err;
     SWalFileInfo* pInfo = &pData[i];
     pField = cJSON_GetObjectItem(pInfoJson, "firstVer");
+    if (!pField) goto _err;
     pInfo->firstVer = atoll(cJSON_GetStringValue(pField));
     pField = cJSON_GetObjectItem(pInfoJson, "lastVer");
+    if (!pField) goto _err;
     pInfo->lastVer = atoll(cJSON_GetStringValue(pField));
     pField = cJSON_GetObjectItem(pInfoJson, "createTs");
+    if (!pField) goto _err;
     pInfo->createTs = atoll(cJSON_GetStringValue(pField));
     pField = cJSON_GetObjectItem(pInfoJson, "closeTs");
+    if (!pField) goto _err;
     pInfo->closeTs = atoll(cJSON_GetStringValue(pField));
     pField = cJSON_GetObjectItem(pInfoJson, "fileSize");
+    if (!pField) goto _err;
     pInfo->fileSize = atoll(cJSON_GetStringValue(pField));
   }
   taosArraySetSize(pArray, sz);
@@ -530,6 +691,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
   pWal->writeCur = sz - 1;
   cJSON_Delete(pRoot);
   return 0;
+
+_err:
+  cJSON_Delete(pRoot);
+  return -1;
 }
 
 static int walFindCurMetaVer(SWal* pWal) {
@@ -565,22 +730,63 @@ static int walFindCurMetaVer(SWal* pWal) {
 int walSaveMeta(SWal* pWal) {
   int  metaVer = walFindCurMetaVer(pWal);
   char fnameStr[WAL_FILE_LEN];
-  walBuildMetaName(pWal, metaVer + 1, fnameStr);
+  char tmpFnameStr[WAL_FILE_LEN];
-  TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE);
+  int  n;
-  if (pMetaFile == NULL) {
+
+  // fsync the idx and log file at first to ensure validity of meta
+  if (taosFsyncFile(pWal->pIdxFile) < 0) {
+    wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
+
+  if (taosFsyncFile(pWal->pLogFile) < 0) {
+    wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  // flush to a tmpfile
+  n = walBuildTmpMetaName(pWal, tmpFnameStr);
+  ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name");
+
+  TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pMetaFile == NULL) {
+    wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
   char* serialized = walMetaSerialize(pWal);
   int   len = strlen(serialized);
   if (len != taosWriteFile(pMetaFile, serialized, len)) {
-    // TODO:clean file
+    wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
-
+    terrno = TAOS_SYSTEM_ERROR(errno);
-    taosCloseFile(&pMetaFile);
+    goto _err;
-    taosRemoveFile(fnameStr);
+  }
-    return -1;
+
+  if (taosFsyncFile(pMetaFile) < 0) {
+    wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (taosCloseFile(&pMetaFile) < 0) {
+    wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  // rename it
+  n = walBuildMetaName(pWal, metaVer + 1, fnameStr);
+  ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name");
+
+  if (taosRenameFile(tmpFnameStr, fnameStr) < 0) {
+    wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
   }
 
-  taosCloseFile(&pMetaFile);
   // delete old file
   if (metaVer > -1) {
     walBuildMetaName(pWal, metaVer, fnameStr);
@@ -588,6 +794,11 @@ int walSaveMeta(SWal* pWal) {
   }
   taosMemoryFree(serialized);
   return 0;
+
+_err:
+  taosCloseFile(&pMetaFile);
+  taosMemoryFree(serialized);
+  return -1;
 }
 
 int walLoadMeta(SWal* pWal) {
@@ -629,6 +840,10 @@ int walLoadMeta(SWal* pWal) {
   }
   // load into fileInfoSet
   int code = walMetaDeserialize(pWal, buf);
+  if (code < 0) {
+    wError("failed to deserialize wal meta. file:%s", fnameStr);
+    terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+  }
   taosCloseFile(&pFile);
   taosMemoryFree(buf);
   return code;

source/libs/wal/src/walMgmt.c

@@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
     return NULL;
   }
 
+  if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    taosMemoryFree(pWal);
+    return NULL;
+  }
+
   // set config
   memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg));
 
@@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
   tstrncpy(pWal->path, path, sizeof(pWal->path));
   if (taosMkDir(pWal->path) != 0) {
     wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
-    taosMemoryFree(pWal);
+    goto _err;
-    return NULL;
   }
 
   // init ref
   pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
   if (pWal->pRefHash == NULL) {
-    taosMemoryFree(pWal);
+    wError("failed to init hash since %s", tstrerror(terrno));
-    return NULL;
+    goto _err;
   }
 
   // open meta
@@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
   pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
   if (pWal->fileInfoSet == NULL) {
     wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
-    taosHashCleanup(pWal->pRefHash);
+    goto _err;
-    taosMemoryFree(pWal);
-    return NULL;
   }
 
   // init status
@@ -131,46 +134,37 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
   pWal->writeHead.head.protoVer = WAL_PROTO_VER;
   pWal->writeHead.magic = WAL_MAGIC;
 
-  if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) {
+  // load meta
-    taosArrayDestroy(pWal->fileInfoSet);
+  (void)walLoadMeta(pWal);
-    taosHashCleanup(pWal->pRefHash);
-    taosMemoryFree(pWal);
-    return NULL;
-  }
-
-  pWal->refId = taosAddRef(tsWal.refSetId, pWal);
-  if (pWal->refId < 0) {
-    taosHashCleanup(pWal->pRefHash);
-    taosThreadMutexDestroy(&pWal->mutex);
-    taosArrayDestroy(pWal->fileInfoSet);
-    taosMemoryFree(pWal);
-    return NULL;
-  }
-
-  walLoadMeta(pWal);
 
   if (walCheckAndRepairMeta(pWal) < 0) {
     wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId);
-    taosHashCleanup(pWal->pRefHash);
+    goto _err;
-    taosRemoveRef(tsWal.refSetId, pWal->refId);
-    taosThreadMutexDestroy(&pWal->mutex);
-    taosArrayDestroy(pWal->fileInfoSet);
-    return NULL;
   }
 
   if (walCheckAndRepairIdx(pWal) < 0) {
     wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId);
-    taosHashCleanup(pWal->pRefHash);
+    goto _err;
-    taosRemoveRef(tsWal.refSetId, pWal->refId);
+  }
-    taosThreadMutexDestroy(&pWal->mutex);
+
-    taosArrayDestroy(pWal->fileInfoSet);
+  // add ref
-    return NULL;
+  pWal->refId = taosAddRef(tsWal.refSetId, pWal);
+  if (pWal->refId < 0) {
+    wError("failed to add ref for Wal since %s", tstrerror(terrno));
+    goto _err;
   }
 
   wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,
          pWal->cfg.fsyncPeriod);
-
   return pWal;
+
+_err:
+  taosArrayDestroy(pWal->fileInfoSet);
+  taosHashCleanup(pWal->pRefHash);
+  taosThreadMutexDestroy(&pWal->mutex);
+  taosMemoryFree(pWal);
+  pWal = NULL;
+  return NULL;
 }
 
 int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
@@ -195,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
 
 void walClose(SWal *pWal) {
   taosThreadMutexLock(&pWal->mutex);
+  (void)walSaveMeta(pWal);
   taosCloseFile(&pWal->pLogFile);
   pWal->pLogFile = NULL;
   taosCloseFile(&pWal->pIdxFile);
   pWal->pIdxFile = NULL;
-  walSaveMeta(pWal);
   taosArrayDestroy(pWal->fileInfoSet);
   pWal->fileInfoSet = NULL;
   taosHashCleanup(pWal->pRefHash);

source/libs/wal/src/walRead.c

@@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
   SWalFileInfo tmpInfo;
   tmpInfo.firstVer = ver;
   SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
-  ASSERT(pRet != NULL);
+  if (pRet == NULL) {
+    wError("failed to find WAL log file with ver:%lld", ver);
+    terrno = TSDB_CODE_WAL_INVALID_VER;
+    return -1;
+  }
   if (pReader->curFileFirstVer != pRet->firstVer) {
     // error code was set inner
     if (walReadChangeFile(pReader, pRet->firstVer) < 0) {
@@ -472,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
       } else {
         terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
       }
-      ASSERT(0);
+      wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s",
+             pReader->pWal->cfg.vgId, ver, terrstr());
       taosThreadMutexUnlock(&pReader->mutex);
       return -1;
     }
@@ -505,6 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
     else {
       terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     }
+    wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s",
+           pReader->pWal->cfg.vgId, ver, terrstr());
     taosThreadMutexUnlock(&pReader->mutex);
     return -1;
   }

source/libs/wal/src/walSeek.c

@@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
   TdFilePtr pIdxTFile, pLogTFile;
   char      fnameStr[WAL_FILE_LEN];
   if (pWal->pLogFile != NULL) {
+    code = taosFsyncFile(pWal->pLogFile);
+    if (code != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
     code = taosCloseFile(&pWal->pLogFile);
     if (code != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
     }
   }
   if (pWal->pIdxFile != NULL) {
+    code = taosFsyncFile(pWal->pIdxFile);
+    if (code != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
     code = taosCloseFile(&pWal->pIdxFile);
     if (code != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);

source/libs/wal/src/walWrite.c

@@ -45,6 +45,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
       if (taosRemoveFile(fnameStr) < 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
+        taosThreadMutexUnlock(&pWal->mutex);
         return -1;
       }
       wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
@@ -53,6 +54,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
       if (taosRemoveFile(fnameStr) < 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
+        taosThreadMutexUnlock(&pWal->mutex);
         return -1;
       }
       wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
@@ -219,10 +221,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   taosCloseFile(&pIdxFile);
   taosCloseFile(&pLogFile);
 
-  taosFsyncFile(pWal->pLogFile);
+  code = walSaveMeta(pWal);
-  taosFsyncFile(pWal->pIdxFile);
+  if (code < 0) {
-
+    wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
-  walSaveMeta(pWal);
+    taosThreadMutexUnlock(&pWal->mutex);
+    return -1;
+  }
 
   // unlock
   taosThreadMutexUnlock(&pWal->mutex);
@@ -394,7 +398,11 @@ int32_t walRollImpl(SWal *pWal) {
 
   pWal->lastRollSeq = walGetSeq();
 
-  walSaveMeta(pWal);
+  code = walSaveMeta(pWal);
+  if (code < 0) {
+    wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
+    goto END;
+  }
 
 END:
   return code;
@@ -550,6 +558,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
 
 void walFsync(SWal *pWal, bool forceFsync) {
   if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
+    wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
+    if (taosFsyncFile(pWal->pIdxFile) < 0) {
+      wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
+             strerror(errno));
+    }
     wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
     if (taosFsyncFile(pWal->pLogFile) < 0) {
       wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),

source/os/src/osEnv.c

@@ -87,11 +87,17 @@ void osUpdate() {
 
 void osCleanup() {}
 
-bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; }
+bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; }
 
-bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; }
+bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; }
 
-bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; }
+bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; }
+
+bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; }
+
+bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; }
+
+bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; }
 
 void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); }
 

source/util/src/tbloomfilter.c

@@ -19,12 +19,12 @@
 #include "taos.h"
 #include "taoserror.h"
 
-#define UNIT_NUM_BITS      64
+#define UNIT_NUM_BITS      64ULL
-#define UNIT_ADDR_NUM_BITS 6
+#define UNIT_ADDR_NUM_BITS 6ULL
 
 static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
   uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
-  uint64_t mask = 1 << (index % UNIT_NUM_BITS);
+  uint64_t mask = 1ULL << (index % UNIT_NUM_BITS);
   uint64_t old = buf[unitIndex];
   buf[unitIndex] |= mask;
   return buf[unitIndex] != old;
@@ -32,7 +32,7 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
 
 static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) {
   uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
-  uint64_t mask = 1 << (index % UNIT_NUM_BITS);
+  uint64_t mask = 1ULL << (index % UNIT_NUM_BITS);
   return buf[unitIndex] & mask;
 }
 

source/util/src/terror.c

@@ -447,12 +447,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND,    "TQ table schema not f
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET,       "TQ no commited offset")
 
 // wal
-TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                "Wal unexpected generic error")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                "WAL unexpected generic error")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED,           "WAL file is corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT,               "WAL size exceeds limit")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER,              "WAL use invalid version")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY,            "WAL out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST,            "WAL log not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH,          "WAL checksum mismatch")
 
 // tfs
 TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG,                 "tfs invalid mount config")

tests/develop-test/5-taos-tools/taosbenchmark/default_json.py

@@ -32,6 +32,10 @@ class TDTestCase:
 
         if "community" in selfPath:
             projPath = selfPath[: selfPath.find("community")]
+        elif "src" in selfPath:
+            projPath = selfPath[: selfPath.find("src")]
+        elif "/tools/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tools/")]
         else:
             projPath = selfPath[: selfPath.find("tests")]
 
@@ -55,11 +59,10 @@ class TDTestCase:
         tdLog.info("%s" % cmd)
         os.system("%s" % cmd)
         tdSql.execute("reset query cache")
-        tdSql.query("select count(*) from (select distinct(tbname) from db.stb)")
+        tdSql.query("show db.tables")
-        tdSql.checkData(0, 0, 10)
+        tdSql.checkRows(10)
         tdSql.query("select count(*) from db.stb")
-        if len(tdSql.queryResult):
+        tdSql.checkData(0, 0, 100)
-            tdLog.exit("query result is %d" % len(tdSql.queryResult))
 
     def stop(self):
         tdSql.close()

tests/develop-test/5-taos-tools/taosbenchmark/json/default.json

@@ -20,6 +20,8 @@
     "super_tables": [{
       "name": "stb",
       "childtable_prefix": "stb_",
+      "childtable_count": 10,
+      "insert_rows": 10,
       "columns": [{"type": "INT"}],
       "tags": [{"type": "INT"}]
     }]

tests/script/coverage_test.sh

@@ -0,0 +1,291 @@
+#!/bin/bash
+
+branch=
+if [ x$1 != x ];then
+        branch=$1
+        echo "Testing branch: $branch"
+else
+        echo "Please enter branch name as a parameter"
+        exit 1
+fi
+
+today=`date +"%Y%m%d"`
+TDENGINE_DIR=/root/pxiao/TDengine
+TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDengine {
+	echo "check if TDengine need build"
+	cd $TDENGINE_DIR
+  	git remote prune origin > /dev/null
+	git remote update > /dev/null
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch`
+	LOCAL_COMMIT=`git rev-parse --short @`
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+
+	# reset counter
+	lcov -d . --zerocounters
+
+
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "repo up-to-date"
+	else
+		echo "repo need to pull"
+	fi
+
+    	git reset --hard
+    	git checkout -- .
+    	git checkout $branch
+	git clean -dfx
+	git pull
+    	git submodule update --init --recursive -f
+ 	
+  	[ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
+  	cd $TDENGINE_DIR/debug
+
+    	echo "rebuild.."
+	LOCAL_COMMIT=`git rev-parse --short @`
+
+	rm -rf *
+	if [ "$branch" == "3.0" ]; then
+		echo "3.0 ============="
+		cmake -DCOVER=true -DBUILD_TEST=true .. 
+	else
+		cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null
+	fi
+	make -j4
+	make install
+}
+
+function runGeneralCaseOneByOne {
+	while read -r line; do
+		if [[ $line =~ ^./test.sh* ]]; then
+			case=`echo $line | grep sim$ | awk '{print $NF}'`
+
+			if [ -n "$case" ]; then
+				date +%F\ %T | tee -a  $TDENGINE_COVERAGE_REPORT  && ./test.sh -f $case > /dev/null 2>&1 && \
+				 echo -e "${GREEN}$case success${NC}" | tee -a  $TDENGINE_COVERAGE_REPORT \
+				|| echo -e "${RED}$case failed${NC}" | tee -a  $TDENGINE_COVERAGE_REPORT
+			fi
+		fi
+	done < $1
+}
+
+function runTestNGCaseOneByOne {
+	while read -r line; do
+		if [[ $line =~ ^taostest* ]]; then
+			case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2`
+			yaml=`echo $line | cut -d' ' -f 2`
+
+			if [ -n "$case" ]; then
+				date +%F\ %T | tee -a  $TDENGINE_COVERAGE_REPORT  && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \
+				 echo -e "${GREEN}$case success${NC}" | tee -a  $TDENGINE_COVERAGE_REPORT \
+				|| echo -e "${RED}$case failed${NC}" | tee -a  $TDENGINE_COVERAGE_REPORT
+			fi
+		fi
+	done < $1
+}
+
+function runTest {
+	echo "run Test"
+
+	if [ "$branch" == "3.0" ]; then
+		echo "start run unit test case ................"
+		echo " $TDENGINE_DIR/debug "
+		cd $TDENGINE_DIR/debug
+		ctest -j12
+		echo "3.0 unit test done"
+	fi
+
+	cd $TDENGINE_DIR/tests/script
+
+	[ -d ../../sim ] && rm -rf ../../sim
+	[ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT
+
+	runGeneralCaseOneByOne jenkins/basic.txt
+
+	sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT
+
+	totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l`
+	if [ "$totalSuccess" -gt "0" ]; then
+		sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###"  $TDENGINE_COVERAGE_REPORT
+	fi
+
+	totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l`
+	if [ "$totalFailed" -ne "0" ]; then
+		sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###"  $TDENGINE_COVERAGE_REPORT
+	fi
+	sed "3G"  $TDENGINE_COVERAGE_REPORT
+
+	stopTaosd
+	echo "run TestNG cases"
+	rm -rf /var/lib/taos/*
+	rm -rf /var/log/taos/*
+        nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
+        sleep 10
+	cd $TDENGINE_DIR/../TestNG/cases
+	runTestNGCaseOneByOne ../scripts/cases.txt
+	echo "TestNG cases done"
+
+	cd $TDENGINE_DIR/tests
+	rm -rf ../sim
+	/root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT
+	
+	
+	sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT
+	totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
+	if [ "$totalPySuccess" -gt "0" ]; then
+                sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###"  $TDENGINE_COVERAGE_REPORT
+        fi
+
+	totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
+        if [ -z $totalPyFailed ]; then
+		sed -i "6i\\n"  $TDENGINE_COVERAGE_REPORT
+        else
+		sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###"  $TDENGINE_COVERAGE_REPORT
+        fi
+
+	echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT
+	# Test Connector
+	stopTaosd
+	nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
+	sleep 10
+
+	cd $TDENGINE_DIR/src/connector/jdbc
+	mvn clean package > /dev/null 2>&1
+	mvn test > jdbc-out.log 2>&1
+  	tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test C Demo
+	stopTaosd
+	$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+	yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test waltest
+	dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
+	walDir=`find $dataDir -name "wal"|head -n1`
+	echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
+	echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
+	if [ -n "$walDir" ]; then
+		yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+	fi
+
+  # run Unit Test
+  echo "Run Unit Test: utilTest, queryTest and cliTest"
+  #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!"
+  #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!"
+  #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!"
+
+	stopTaosd
+
+	cd $TDENGINE_DIR/tests/script
+	find . -name '*.sql' | xargs rm -f
+
+	cd $TDENGINE_DIR/tests/pytest
+	find . -name '*.sql' | xargs rm -f
+}
+
+function lcovFunc {
+	echo "collect data by lcov"
+	cd $TDENGINE_DIR
+
+	# collect data
+	lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
+
+	# remove exclude paths
+	if [ "$branch" == "3.0" ]; then
+		lcov --remove coverage.info \
+			'*/contrib/*' '*/tests/*' '*/test/*'\
+		       	'*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\
+			'*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\
+		       	'*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\
+		       	'*/tthread.c' '*/tversion.c'\
+		       	--rc lcov_branch_coverage=1 -o coverage.info
+	else	
+		lcov --remove coverage.info \
+    		'*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\
+    		--rc lcov_branch_coverage=1 -o coverage.info
+	fi
+
+
+	# generate result
+	echo "generate result"
+	lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# push result to coveralls.io
+	echo "push result to coveralls.io"
+	/usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT
+
+	#/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT
+	#cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT
+	cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT
+	
+}
+
+function sendReport {
+	echo "send report"
+	receiver="develop@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDENGINE_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT
+	BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT`
+	echo -e "from: <support@taosdata.com>\nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \
+	/usr/sbin/ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+	echo "Stop taosd start"
+        systemctl stop taosd
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+    pkill -TERM -x taosd
+    sleep 1
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+	echo "Stop tasod end"
+}
+
+function runTestRandomFail {
+  exec_random_fail_sh=$1
+  default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh
+	[ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1
+
+  dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py
+  default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py
+	[ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1
+
+  runTest NoRandomFail
+}
+
+WORK_DIR=/root/pxiao
+
+date >> $WORK_DIR/cron.log
+echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log
+
+stopTaosd
+buildTDengine
+
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh 
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh 
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh 
+
+runTest
+
+lcovFunc
+#sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log

tests/script/jenkins/basic.txt

@@ -155,7 +155,7 @@
 ./test.sh -f tsim/parser/select_with_tags.sim
 ./test.sh -f tsim/parser/selectResNum.sim
 ./test.sh -f tsim/parser/set_tag_vals.sim
-./test.sh -f tsim/parser/single_row_in_tb.sim
+# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim
 ./test.sh -f tsim/parser/sliding.sim
 ./test.sh -f tsim/parser/slimit_alter_tags.sim
 ./test.sh -f tsim/parser/slimit.sim
@@ -194,7 +194,7 @@
 ./test.sh -f tsim/mnode/basic1.sim
 ./test.sh -f tsim/mnode/basic2.sim
 ./test.sh -f tsim/mnode/basic3.sim
-# TD-17919 ./test.sh -f tsim/mnode/basic4.sim
+./test.sh -f tsim/mnode/basic4.sim
 ./test.sh -f tsim/mnode/basic5.sim
 
 # ---- show ----

tests/script/tsim/mnode/basic1.sim

@@ -42,6 +42,7 @@ sql_error drop mnode on dnode 1
 print =============== create mnode 2
 sql create mnode on dnode 2
 
+print =============== create mnode 2 finished
 $x = 0
 step2: 
 	$x = $x + 1
@@ -69,9 +70,10 @@ if $data(2)[2] != follower then
   goto step2
 endi
 
-sleep 2000
 print ============ drop mnode 2
 sql drop mnode on dnode 2
+
+print ============ drop mnode 2 finished
 sql select * from information_schema.ins_mnodes
 if $rows != 1 then 
   return -1
@@ -109,6 +111,8 @@ sleep 2000
 
 print =============== create mnodes
 sql create mnode on dnode 2
+
+print =============== create mnode 2 finished
 sql select * from information_schema.ins_mnodes
 if $rows != 2 then 
   return -1

tests/script/tsim/mnode/basic4.sim

@@ -45,49 +45,12 @@ endi
 if $data(2)[4] != ready then
   goto step2
 endi
-
+if $data(3)[4] != ready then
-system sh/exec.sh -n dnode3 -s stop
+  goto step2
-sql_error create mnode on dnode 3
-
-print =============== step3: select * from information_schema.ins_mnodes
-
-$x = 0
-step3: 
-	$x = $x + 1
-	sleep 1000
-	if $x == 10 then
-		return -1
-	endi
-sql select * from information_schema.ins_mnodes -x step3
-print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
-print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4]
-
-$leaderNum = 0
-if $data(1)[2] == leader then
-  $leaderNum = 1
-endi
-if $data(2)[2] == leader then
-  $leaderNum = 1
-endi
-if  $leaderNum == 0 then
-  goto step3
-endi
-if $data(3)[2] != offline then
-  goto step3
-endi
-if $data(1)[3] != ready then
-  goto step3
-endi
-if $data(2)[3] != ready then
-  goto step3
-endi
-if $data(3)[3] != creating then
-  goto step3
 endi
 
-print =============== step4: start dnode3
+print =============== step4: create mnode 3
-system sh/exec.sh -n dnode3 -s start
+sql create mnode on dnode 3
 
 $x = 0
 step4: 
@@ -159,12 +122,11 @@ endi
 if $data(2)[3] != ready then
   goto step5
 endi
-if $data(3)[3] != dropping then
-  goto step5
-endi
 
 print =============== step6: start dnode3
 system sh/exec.sh -n dnode3 -s start
+sql drop mnode on dnode 1 -x step60
+step60:
 
 $x = 0
 step6: 

tests/script/tsim/mnode/basic5.sim

@@ -232,12 +232,11 @@ endi
 if  $leaderNum != 1 then
   goto step81
 endi
-if $data(1)[3] != dropping then
-  goto step81
-endi
 
 print =============== step9: start mnode1 and wait it dropped
 system sh/exec.sh -n dnode1 -s start
+sql drop mnode on dnode 1 -x step90
+step90:
 
 $x = 0
 step91: 

tests/script/tsim/valgrind/checkError3.sim

@@ -62,7 +62,16 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start -v
 
 print =============== stepa: query data
-sql select * from c1
+
+$x = 0
+steps: 
+	$x = $x + 1
+	sleep 500
+	if $x == 50 then
+		return -1
+	endi
+sql select * from c1 -x steps
+
 sql select * from stb
 sql select * from stb_1
 sql select ts, c1, c2, c3 from c1

tests/script/tsim/valgrind/checkError6.sim

@@ -158,9 +158,15 @@ print =============== restart
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start -v
 
-sleep 1000
+$x = 0
+steps: 
+	$x = $x + 1
+	sleep 500
+	if $x == 50 then
+		return -1
+	endi
+sql select avg(tbcol) as c from stb -x steps
 
-sql select avg(tbcol) as c from stb
 sql select avg(tbcol) as c from stb where ts <= 1601481840000
 sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000
 sql select avg(tbcol) as c from stb interval(1m)

tests/system-test/1-insert/alter_database.py

@@ -0,0 +1,58 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(),logSql)
+        self.buffer_boundary = [3,4097,8193,12289,16384]
+        self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96]
+        # pages_boundary >= 64
+        self.pages_boundary = [64,128,512]
+        self.pages_error = [self.pages_boundary[0]-1]
+    def alter_buffer(self):
+        tdSql.execute('create database db')
+        for buffer in self.buffer_boundary:
+            tdSql.execute(f'alter database db buffer {buffer}')
+            tdSql.query('select * from information_schema.ins_databases where name = "db"')
+            tdSql.checkEqual(tdSql.queryResult[0][8],buffer)    
+        tdSql.execute('drop database db')
+        tdSql.execute('create database db vgroups 10')
+        for buffer in self.buffer_error:
+            tdSql.error(f'alter database db buffer {buffer}')
+        tdSql.execute('drop database db')
+    
+    def alter_pages(self):
+        tdSql.execute('create database db')
+        for pages in self.pages_boundary:
+            tdSql.execute(f'alter database db pages {pages}')
+            tdSql.query('select * from information_schema.ins_databases where name = "db"')
+            tdSql.checkEqual(tdSql.queryResult[0][10],pages)
+        tdSql.execute('drop database db')
+        tdSql.execute('create database db')
+        tdSql.query('select * from information_schema.ins_databases where name = "db"')
+        self.pages_error.append(tdSql.queryResult[0][10])
+        for pages in self.pages_error:
+            tdSql.error(f'alter database db pages {pages}')
+        tdSql.execute('drop database db')
+
+    def run(self):
+        tdSql.error('create database db1 vgroups 10 buffer 12289')
+        self.alter_buffer()
+        self.alter_pages()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/create_wrong_topic.py

@@ -0,0 +1,77 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+from util.common import *
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.sqlset import *
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(),logSql)
+        self.setsql = TDSetSql()
+        self.rowNum = 10
+        self.ts = 1537146000000
+        self.binary_str = 'taosdata'
+        self.nchar_str = '涛思数据'
+        self.column_dict = {
+            'ts'  : 'timestamp',
+            'col1': 'tinyint',
+            'col2': 'smallint',
+            'col3': 'int',
+            'col4': 'bigint',
+            'col5': 'tinyint unsigned',
+            'col6': 'smallint unsigned',
+            'col7': 'int unsigned',
+            'col8': 'bigint unsigned',
+            'col9': 'float',
+            'col10': 'double',
+            }
+        self.error_topic = ['avg','count','spread','stddev','sum','hyperloglog']
+    def insert_data(self,column_dict,tbname,row_num):
+        insert_sql = self.setsql.set_insertsql(column_dict,tbname)
+        for i in range(row_num):
+            insert_list = []
+            self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts)
+    def wrong_topic(self):
+        tdSql.prepare()
+        tdSql.execute('use db')
+        stbname = f'db.{tdCom.getLongName(5, "letters")}'
+        tag_dict = {
+            't0':'int'
+        }
+        tag_values = [
+            f'1'
+            ]
+        tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict))
+        tdSql.execute(f"create table {stbname}_tb1 using {stbname} tags({tag_values[0]})")
+        self.insert_data(self.column_dict,f'{stbname}_tb1',self.rowNum)
+        for column in self.column_dict.keys():
+            for func in self.error_topic:
+                if func.lower() != 'count' and column.lower() != 'ts':
+                    tdSql.error(f'create topic tpn as select {func}({column}) from {stbname}')
+                elif func.lower() == 'count' :
+                    tdSql.error(f'create topic tpn as select {func}(*) from {stbname}')
+        for column in self.column_dict.keys():
+            if column.lower() != 'ts':
+                tdSql.error(f'create topic tpn as select apercentile({column},50) from {stbname}')
+                tdSql.error(f'create topic tpn as select leastquares({column},1,1) from {stbname}_tb1')
+                tdSql.error(f'create topic tpn as select HISTOGRAM({column},user_input,[1,3,5,7],0) from {stbname}')
+                tdSql.error(f'create topic tpn as select percentile({column},1) from {stbname}_tb1')
+        pass
+    def run(self):
+        self.wrong_topic()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqAutoCreateTbl.py

@@ -52,7 +52,7 @@ class TDTestCase:
         paraDict['rowsPerTbl'] = self.rowsPerTbl
 
         tmqCom.initConsumerTable()
-        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1)
         tdLog.info("create stb")
         tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
         # tdLog.info("create ctb")

tests/system-test/fulltest.sh

@@ -2,7 +2,7 @@
 set -e
 set -x
 
-#python3 ./test.py -f 0-others/taosShell.py
+python3 ./test.py -f 0-others/taosShell.py
 python3 ./test.py -f 0-others/taosShellError.py
 python3 ./test.py -f 0-others/taosShellNetChk.py
 python3 ./test.py -f 0-others/telemetry.py
@@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/sysinfo.py
 python3 ./test.py -f 0-others/user_control.py
 python3 ./test.py -f 0-others/fsync.py
 python3 ./test.py  -f 0-others/compatibility.py   
-
+python3 ./test.py -f 1-insert/alter_database.py
 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
 python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py
 python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py
@@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py  -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
-# unstable python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py  -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3
@@ -248,8 +248,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 -
 
 python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py
-# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
+python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
-# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3
+python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3
 
 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py  -N 5 -M 3
 python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py  -N 5 -M 3
@@ -279,7 +279,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_
 python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py  -N 4 -M 1
 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1
 
-
+python3 ./test.py -f 7-tmq/create_wrong_topic.py
 python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
 python3 ./test.py -f 7-tmq/basic5.py
 python3 ./test.py -f 7-tmq/subscribeDb.py

tools/shell/src/shellArguments.c

@@ -22,7 +22,7 @@
 #define TAOS_CONSOLE_PROMPT_HEADER "taos> "
 #define TAOS_CONSOLE_PROMPT_CONTINUE "   -> "
 
-#define SHELL_HOST     "The auth string to use when connecting to the server."
+#define SHELL_HOST     "TDengine server FQDN to connect. The default host is localhost."
 #define SHELL_PORT     "The TCP/IP port number to use for the connection."
 #define SHELL_USER     "The user name to use when connecting to the server."
 #define SHELL_PASSWORD "The password to use when connecting to the server."

tools/shell/src/shellAuto.c

@@ -325,24 +325,23 @@ int        cntDel = 0;        // delete byte count after next press tab
 
 // show auto tab introduction
 void printfIntroduction() {
-  printf("   ****************************  How To Use TAB Key  ********************************\n");
+  printf("   ******************************  Tab Completion  **********************************\n");
-  printf("   *   TDengine Command Line supports pressing TAB key to complete word,            *\n");
+  printf("   *   The TDengine CLI supports tab completion for a variety of items,             *\n");
-  printf("   *   including database name, table name, function name and keywords.             *\n");
+  printf("   *   including database names, table names, function names and keywords.          *\n");
-  printf("   *   Press TAB key anywhere, you'll get surprise.                                 *\n");
+  printf("   *   The full list of shortcut keys is as follows:                                *\n");
-  printf("   *   KEYBOARD SHORTCUT:                                                           *\n");
+  printf("   *    [ TAB ]        ......  complete the current word                            *\n");
-  printf("   *    [ TAB ]        ......  Complete the word or show help if no input           *\n");
+  printf("   *                   ......  if used on a blank line, display all valid commands  *\n");
-  printf("   *    [ Ctrl + A ]   ......  move cursor to [A]head of line                       *\n");
+  printf("   *    [ Ctrl + A ]   ......  move cursor to the st[A]rt of the line               *\n");
-  printf("   *    [ Ctrl + E ]   ......  move cursor to [E]nd of line                         *\n");
+  printf("   *    [ Ctrl + E ]   ......  move cursor to the [E]nd of the line                 *\n");
-  printf("   *    [ Ctrl + W ]   ......  move cursor to line of middle                        *\n");
+  printf("   *    [ Ctrl + W ]   ......  move cursor to the middle of the line                *\n");
-  printf("   *    [ Ctrl + L ]   ......  clean screen                                         *\n");
+  printf("   *    [ Ctrl + L ]   ......  clear the entire screen                              *\n");
-  printf("   *    [ Ctrl + K ]   ......  clean after cursor                                   *\n");
+  printf("   *    [ Ctrl + K ]   ......  clear the screen after the cursor                    *\n");
-  printf("   *    [ Ctrl + U ]   ......  clean before cursor                                  *\n");
+  printf("   *    [ Ctrl + U ]   ......  clear the screen before the cursor                   *\n");
-  printf("   *                                                                                *\n");
   printf("   **********************************************************************************\n\n");
 }
 
 void showHelp() {
-  printf("\nThe following are supported commands for TDengine Command Line:");
+  printf("\nThe TDengine CLI supports the following commands:");
   printf(
       "\n\
   ----- A ----- \n\

tools/shell/src/shellEngine.c

@@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) {
     printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6);
 
     // call auto tab
-    callbackAutoTab(command, pSql, false);    
+    callbackAutoTab(command, NULL, false);
   }
 
   printf("\r\n");