diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 7bde332c8c..73409a950a 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG d58230c
+ GIT_TAG 4d02980
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md
index d95086d4c4..6295d8553d 100644
--- a/docs/en/14-reference/05-taosbenchmark.md
+++ b/docs/en/14-reference/05-taosbenchmark.md
@@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js
- **name**: specify the name of the database.
-- **drop**: indicate whether to delete the database before inserting. The default is true.
+- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop.
#### Stream processing related configuration parameters
diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py
index cee036454e..836beb2417 100644
--- a/docs/examples/python/tmq_example.py
+++ b/docs/examples/python/tmq_example.py
@@ -1,6 +1,58 @@
import taos
-from taos.tmq import TaosConsumer
-consumer = TaosConsumer('topic_ctb_column', group_id='vg2')
-for msg in consumer:
- for row in msg:
- print(row)
+from taos.tmq import *
+
+conn = taos.connect()
+
+print("init")
+conn.execute("drop database if exists py_tmq")
+conn.execute("create database if not exists py_tmq vgroups 2")
+conn.select_db("py_tmq")
+conn.execute(
+ "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)"
+)
+conn.execute("create table if not exists tb1 using stb1 tags(1)")
+conn.execute("create table if not exists tb2 using stb1 tags(2)")
+conn.execute("create table if not exists tb3 using stb1 tags(3)")
+
+print("create topic")
+conn.execute("drop topic if exists topic_ctb_column")
+conn.execute(
+ "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1"
+)
+
+print("build consumer")
+conf = TaosTmqConf()
+conf.set("group.id", "tg2")
+conf.set("td.connect.user", "root")
+conf.set("td.connect.pass", "taosdata")
+conf.set("enable.auto.commit", "true")
+
+
+def tmq_commit_cb_print(tmq, resp, offset, param=None):
+ print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}")
+
+
+conf.set_auto_commit_cb(tmq_commit_cb_print, None)
+tmq = conf.new_consumer()
+
+print("build topic list")
+
+topic_list = TaosTmqList()
+topic_list.append("topic_ctb_column")
+
+print("basic consume loop")
+tmq.subscribe(topic_list)
+
+sub_list = tmq.subscription()
+
+print("subscribed topics: ", sub_list)
+
+while 1:
+ res = tmq.poll(1000)
+ if res:
+ topic = res.get_topic_name()
+ vg = res.get_vgroup_id()
+ db = res.get_db_name()
+ print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}")
+ for row in res:
+ print(row)
diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md
index 7da8e9f331..1ab7f4f016 100644
--- a/docs/zh/12-taos-sql/29-changes.md
+++ b/docs/zh/12-taos-sql/29-changes.md
@@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明"
| 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
| 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
| 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
+
+
+## SCHEMALESS 变更
+
+| # | **元素** | **差异性
** | **说明** |
+| - | :------- | :-------- | :------- |
+| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。
diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md
index 9f4f728f78..e067c646b2 100644
--- a/docs/zh/14-reference/05-taosbenchmark.md
+++ b/docs/zh/14-reference/05-taosbenchmark.md
@@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
- **name** : 数据库名。
-- **drop** : 插入前是否删除数据库，默认为 true。
+- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。
#### 流式计算相关配置参数
diff --git a/examples/c/demoapi.c b/examples/c/demoapi.c
index 88d924e97c..18f1b5a059 100644
--- a/examples/c/demoapi.c
+++ b/examples/c/demoapi.c
@@ -1,5 +1,17 @@
-// C api call sequence demo
-// to compile: gcc -o apidemo apidemo.c -ltaos
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
#include
#include
@@ -13,23 +25,23 @@
#include "taos.h"
#define debugPrint(fmt, ...) \
- do { if (g_args.debug_print || g_args.verbose_print) \
- fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0)
+ do { if (g_args.debug_print || g_args.verbose_print) {\
+ fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0)
#define warnPrint(fmt, ...) \
do { fprintf(stderr, "\033[33m"); \
fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \
- fprintf(stderr, "\033[0m"); } while(0)
+ fprintf(stderr, "\033[0m"); } while (0)
#define errorPrint(fmt, ...) \
do { fprintf(stderr, "\033[31m"); \
fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \
- fprintf(stderr, "\033[0m"); } while(0)
+ fprintf(stderr, "\033[0m"); } while (0)
#define okPrint(fmt, ...) \
do { fprintf(stderr, "\033[32m"); \
fprintf(stderr, "OK: "fmt, __VA_ARGS__); \
- fprintf(stderr, "\033[0m"); } while(0)
+ fprintf(stderr, "\033[0m"); } while (0)
int64_t g_num_of_tb = 2;
int64_t g_num_of_rec = 3;
@@ -75,14 +87,15 @@ static void prepare_data(TAOS* taos) {
taos_free_result(res);
taosMsleep(100);
if (taos_select_db(taos, "test")) {
- errorPrint("%s() LN%d: error no: %d, reason: %s\n",
- __func__, __LINE__, taos_errno(res), taos_errstr(res));
- taos_free_result(res);
+ errorPrint("%s() LN%d: taos_select_db() failed\n",
+ __func__, __LINE__);
return;
}
char command[1024] = {0};
- sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));");
+ sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, "
+ "bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, "
+ "city binary(20), dist nchar(20), street binary(20));");
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
okPrint("%s created\n", "meters");
@@ -122,12 +135,11 @@ static void prepare_data(TAOS* taos) {
sprintf(command, "insert into t%"PRId64" "
"values(%" PRId64 ", %f, %"PRId64", "
"'%c%d', '%s%c%d', '%c%d')",
- i, 1650000000000+j, (float)j, j,
+ i, 1650000000000+j, j * 1.0, j,
'a'+(int)j%25, rand(),
// "涛思", 'z' - (int)j%25, rand(),
"TAOS", 'z' - (int)j%25, rand(),
- 'b' - (int)j%25, rand()
- );
+ 'b' - (int)j%25, rand());
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
affected = taos_affected_rows(res);
@@ -155,8 +167,7 @@ static void prepare_data(TAOS* taos) {
i, 1650000000000+j+1, (float)j, j,
'a'+(int)j%25, rand(),
"数据", 'z' - (int)j%25, rand(),
- 'b' - (int)j%25, rand()
- );
+ 'b' - (int)j%25, rand());
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
affected = taos_affected_rows(res);
@@ -184,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
}
if (block) {
- warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
+ warnPrint("%s", "call taos_fetch_block(), "
+ "don't call taos_fetch_lengths()\n");
int rows = 0;
while ((rows = taos_fetch_block(res, &row))) {
int *lengths = taos_fetch_lengths(res);
@@ -195,7 +207,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d type is %d, no need get offset\n",
f, fields[f].type);
for (int64_t c = 0; c < rows; c++) {
- switch(fields[f].type) {
+ switch (fields[f].type) {
case TSDB_DATA_TYPE_TIMESTAMP:
if (taos_is_null(res, c, f)) {
printf("col%d, row: %"PRId64" "
@@ -204,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %"PRId64"\n",
f, c,
- *(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
+ *(int64_t*)((char*)(row[f])
+ +c*sizeof(int64_t)));
}
break;
@@ -216,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %d\n",
f, c,
- *(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
+ *(int32_t*)((char*)(row[f])
+ +c*sizeof(int32_t)));
}
break;
@@ -228,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %f\n",
f, c,
- *(float*)((char*)(row[f])+c*sizeof(float)));
+ *(float*)((char*)(row[f])
+ +c*sizeof(float)));
}
break;
@@ -243,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
if (offsets) {
for (int c = 0; c < rows; c++) {
if (offsets[c] != -1) {
- int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
+ int length = *(int16_t*)((char*)(row[f])
+ + offsets[c]);
char *buf = calloc(1, length + 1);
- strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
- printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
+ strncpy(buf, (char *)((char*)(row[f])
+ + offsets[c] + 2), length);
+ printf("row: %d, col: %d, offset: %d, "
+ "length: %d, content: %s\n",
c, f, offsets[c], length, buf);
free(buf);
} else {
- printf("row: %d, col: %d, offset: -1, means content is NULL\n",
+ printf("row: %d, col: %d, offset: -1, "
+ "means content is NULL\n",
c, f);
}
}
@@ -272,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
int* lengths = taos_fetch_lengths(res);
if (lengths) {
for (int c = 0; c < num_fields; c++) {
- printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n",
+ printf("row: %"PRId64", col: %d, is_null: %s, "
+ "length of column %d is %d\n",
num_rows, c,
taos_is_null(res, num_rows, c)?"True":"False",
c, lengths[c]);
@@ -282,7 +302,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
__func__, __LINE__, tbname);
}
- num_rows ++;
+ num_rows++;
}
}
@@ -327,7 +347,8 @@ int main(int argc, char *argv[]) {
#endif
TAOS* taos = taos_connect(host, user, passwd, "", 0);
if (taos == NULL) {
- printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));
+ printf("\033[31mfailed to connect to db, reason:%s\033[0m\n",
+ taos_errstr(taos));
exit(1);
}
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 9f21ee007f..5e2ca8896a 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) {
if (pRebInfo == NULL) {
return NULL;
}
- strcpy(pRebInfo->key, key);
+ tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN);
pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t));
if (pRebInfo->lostConsumers == NULL) {
goto _err;
diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h
index 0d43539629..cdb1642a5c 100644
--- a/include/dnode/mnode/mnode.h
+++ b/include/dnode/mnode/mnode.h
@@ -30,9 +30,10 @@ typedef struct SMnode SMnode;
typedef struct {
int32_t dnodeId;
- bool standby;
bool deploy;
- SReplica replica;
+ int8_t selfIndex;
+ int8_t numOfReplicas;
+ SReplica replicas[TSDB_MAX_REPLICA];
SMsgCb msgCb;
} SMnodeOpt;
diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h
index 411a63a379..0f8220f19c 100644
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@@ -211,7 +211,7 @@ typedef struct SSyncInfo {
int32_t syncInit();
void syncCleanUp();
-int64_t syncOpen(const SSyncInfo* pSyncInfo);
+int64_t syncOpen(SSyncInfo* pSyncInfo);
void syncStart(int64_t rid);
void syncStop(int64_t rid);
int32_t syncSetStandby(int64_t rid);
@@ -233,7 +233,7 @@ const char* syncStr(ESyncState state);
bool syncIsRestoreFinish(int64_t rid);
int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot);
-int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg);
+int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg);
// build SRpcMsg, need to call syncPropose with SRpcMsg
int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg);
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index de31a970df..adf244e32a 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -43,6 +43,7 @@ extern "C" {
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL
#define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3)
+#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE)
typedef enum {
TAOS_WAL_WRITE = 1,
diff --git a/include/os/osEnv.h b/include/os/osEnv.h
index 293d9d17f8..d4e94d6173 100644
--- a/include/os/osEnv.h
+++ b/include/os/osEnv.h
@@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace;
void osDefaultInit();
void osUpdate();
void osCleanup();
+
bool osLogSpaceAvailable();
bool osDataSpaceAvailable();
bool osTempSpaceAvailable();
+
+bool osLogSpaceSufficient();
+bool osDataSpaceSufficient();
+bool osTempSpaceSufficient();
+
void osSetTimezone(const char *timezone);
void osSetSystemLocale(const char *inLocale, const char *inCharSet);
@@ -59,4 +65,4 @@ void osSetSystemLocale(const char *inLocale, const char *inCharSet);
}
#endif
-#endif /*_TD_OS_ENV_H_*/
\ No newline at end of file
+#endif /*_TD_OS_ENV_H_*/
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 6bc0e0e7dd..bbddb539c6 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -450,6 +450,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003)
#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004)
#define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005)
+#define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006)
// tfs
#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201)
diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh
index b56a5a203d..d0e94e3eb3 100755
--- a/packaging/tools/make_install.sh
+++ b/packaging/tools/make_install.sh
@@ -168,7 +168,7 @@ function install_bin() {
${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || :
- [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || :
+ [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
[ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
@@ -182,21 +182,21 @@ function install_bin() {
${csudo}chmod 0555 ${install_main_dir}/bin/*
#Make link
- [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
- [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
- [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || :
- [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || :
- [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
- [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
- [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || :
- [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || :
- [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
- [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
+ [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || :
else
${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || :
- [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || :
+ [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
[ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
@@ -206,14 +206,14 @@ function install_bin() {
${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || :
${csudo}chmod 0555 ${install_main_dir}/bin/*
#Make link
- [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
- [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
- [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || :
- [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || :
- [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
- [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || :
- [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || :
- [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
+ [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
+ [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || :
+ [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || :
fi
}
@@ -238,7 +238,7 @@ function install_jemalloc() {
if [ -f "${binary_dir}/build/lib/libjemalloc.so.2" ]; then
${csudo}/usr/bin/install -c -d /usr/local/lib
${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib
- ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+ ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so > /dev/null 2>&1
${csudo}/usr/bin/install -c -d /usr/local/lib
[ -f ${binary_dir}/build/lib/libjemalloc.a ] &&
${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib
@@ -274,8 +274,8 @@ function install_avro() {
if [ -f "${binary_dir}/build/$1/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then
${csudo}/usr/bin/install -c -d /usr/local/$1
${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.so.23.0.0 /usr/local/$1
- ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23
- ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so
+ ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 > /dev/null 2>&1
+ ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so > /dev/null 2>&1
${csudo}/usr/bin/install -c -d /usr/local/$1
[ -f ${binary_dir}/build/$1/libavro.a ] &&
${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.a /usr/local/$1
@@ -304,11 +304,11 @@ function install_lib() {
${install_main_dir}/driver &&
${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber}
- ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
- ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
+ ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 > /dev/null 2>&1
+ ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so > /dev/null 2>&1
if [ -d "${lib64_link_dir}" ]; then
- ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1
- ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so
+ ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 > /dev/null 2>&1
+ ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so > /dev/null 2>&1
fi
if [ -f ${binary_dir}/build/lib/libtaosws.so ]; then
@@ -316,23 +316,23 @@ function install_lib() {
${install_main_dir}/driver &&
${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||:
- ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
+ ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so > /dev/null 2>&1 || :
fi
else
${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \
${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
${csudo}ln -sf ${install_main_dir}/driver/libtaos.${verNumber}.dylib \
- ${lib_link_dir}/libtaos.1.dylib || :
+ ${lib_link_dir}/libtaos.1.dylib > /dev/null 2>&1 || :
- ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib || :
+ ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib > /dev/null 2>&1 || :
if [ -f ${binary_dir}/build/lib/libtaosws.dylib ]; then
${csudo}cp ${binary_dir}/build/lib/libtaosws.dylib \
${install_main_dir}/driver &&
${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.dylib ||:
- ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib || :
+ ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib > /dev/null 2>&1 || :
fi
fi
@@ -346,6 +346,7 @@ function install_lib() {
}
function install_header() {
+ ${csudo}mkdir -p ${inc_link_dir}
${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||:
${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \
@@ -353,13 +354,13 @@ function install_header() {
if [ -f ${binary_dir}/build/include/taosws.h ]; then
${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||:
- ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||:
+ ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h > /dev/null 2>&1 ||:
fi
- ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
- ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
- ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
- ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h
+ ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h > /dev/null 2>&1
+ ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h > /dev/null 2>&1
+ ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h > /dev/null 2>&1
+ ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h > /dev/null 2>&1
${csudo}chmod 644 ${install_main_dir}/include/*
}
@@ -374,7 +375,7 @@ function install_config() {
${csudo}cp -f ${script_dir}/../cfg/${configFile} \
${cfg_install_dir}/${configFile}.${verNumber}
${csudo}ln -s ${cfg_install_dir}/${configFile} \
- ${install_main_dir}/cfg/${configFile}
+ ${install_main_dir}/cfg/${configFile} > /dev/null 2>&1
else
${csudo}cp -f ${script_dir}/../cfg/${configFile} \
${cfg_install_dir}/${configFile}.${verNumber}
@@ -395,7 +396,7 @@ function install_taosadapter_config() {
${cfg_install_dir}/taosadapter.toml.${verNumber} || :
[ -f ${cfg_install_dir}/taosadapter.toml ] &&
${csudo}ln -s ${cfg_install_dir}/taosadapter.toml \
- ${install_main_dir}/cfg/taosadapter.toml || :
+ ${install_main_dir}/cfg/taosadapter.toml > /dev/null 2>&1 || :
else
if [ -f "${binary_dir}/test/cfg/taosadapter.toml" ]; then
${csudo}cp -f ${binary_dir}/test/cfg/taosadapter.toml \
@@ -408,12 +409,12 @@ function install_taosadapter_config() {
function install_log() {
${csudo}rm -rf ${log_dir} || :
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
- ${csudo}ln -s ${log_dir} ${install_main_dir}/log
+ ${csudo}ln -s ${log_dir} ${install_main_dir}/log > /dev/null 2>&1
}
function install_data() {
${csudo}mkdir -p ${data_dir} && ${csudo}chmod 777 ${data_dir}
- ${csudo}ln -s ${data_dir} ${install_main_dir}/data
+ ${csudo}ln -s ${data_dir} ${install_main_dir}/data > /dev/null 2>&1
}
function install_connector() {
@@ -533,7 +534,7 @@ function install_taosadapter_service() {
function install_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist
- ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || :
+ ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
}
function install_service() {
diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index aa725a4988..fa0e3212a3 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes;
pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes;
pTscObj->connId = pRsp->query->connId;
- tscTrace("conn %d hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes,
+ tscTrace("conn %u hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes,
pTscObj->pAppInfo->totalDnodes);
if (pRsp->query->killRid) {
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 940dd745c8..b494140da5 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -186,7 +186,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
STscObj* pTscObj = (*pRequest)->pTscObj;
if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
sizeof((*pRequest)->self))) {
- tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self,
+ tscError("%" PRIx64 " failed to add to request container, reqId:0x%" PRIu64 ", conn:%" PRIx64 ", %s", (*pRequest)->self,
(*pRequest)->requestId, pTscObj->id, sql);
taosMemoryFree(param);
@@ -371,7 +371,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
pInfo->pQnodeList = taosArrayDup(pNodeList);
taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad);
tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId,
- taosArrayGetSize(pInfo->pQnodeList));
+ (int)taosArrayGetSize(pInfo->pQnodeList));
}
taosThreadMutexUnlock(&pInfo->qnodeMutex);
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 7d8026f314..6126817ece 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -146,7 +146,7 @@ void taos_close(TAOS *taos) {
int taos_errno(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
- if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
return terrno;
}
@@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) {
return 0;
}
- return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL
- : ((SRequestObj *)res)->code;
+ return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code;
}
const char *taos_errstr(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
- if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
return (const char *)tstrerror(terrno);
}
@@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) {
if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
return pRequest->msgBuf;
} else {
- return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL)
+ return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY)
: (const char *)tstrerror(pRequest->code);
}
}
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index a257335931..1dc319be94 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -410,6 +410,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
SDecoder decoder = {0};
SVAlterTbReq vAlterTbReq = {0};
char* string = NULL;
+ cJSON* json = NULL;
// decode
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -419,7 +420,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
goto _exit;
}
- cJSON* json = cJSON_CreateObject();
+ json = cJSON_CreateObject();
if (json == NULL) {
goto _exit;
}
@@ -524,6 +525,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
SDecoder decoder = {0};
SVDropStbReq req = {0};
char* string = NULL;
+ cJSON* json = NULL;
// decode
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -533,7 +535,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) {
goto _exit;
}
- cJSON* json = cJSON_CreateObject();
+ json = cJSON_CreateObject();
if (json == NULL) {
goto _exit;
}
@@ -556,6 +558,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
SDecoder decoder = {0};
SVDropTbBatchReq req = {0};
char* string = NULL;
+ cJSON* json = NULL;
// decode
void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead));
@@ -565,7 +568,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) {
goto _exit;
}
- cJSON* json = cJSON_CreateObject();
+ json = cJSON_CreateObject();
if (json == NULL) {
goto _exit;
}
@@ -684,7 +687,7 @@ end:
static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
SVDropStbReq req = {0};
- SDecoder coder;
+ SDecoder coder = {0};
SMDropStbReq pReq = {0};
int32_t code = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = NULL;
@@ -1212,6 +1215,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
int32_t code = TSDB_CODE_SUCCESS;
STableMeta* pTableMeta = NULL;
SQuery* pQuery = NULL;
+ SSubmitReq* subReq = NULL;
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT);
if (!pRequest) {
@@ -1228,8 +1232,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
}
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
- strcpy(pName.dbname, pRequest->pDb);
- strcpy(pName.tname, tbname);
+ tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname));
+ tstrncpy(pName.tname, tbname, sizeof(pName.tname));
struct SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
@@ -1278,7 +1282,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize;
int32_t totalLen = sizeof(SSubmitReq) + submitLen;
- SSubmitReq* subReq = taosMemoryCalloc(1, totalLen);
+ subReq = taosMemoryCalloc(1, totalLen);
SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq));
void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk));
STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen);
@@ -1352,6 +1356,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
if (NULL == pQuery) {
uError("create SQuery error");
code = TSDB_CODE_OUT_OF_MEMORY;
+ taosMemoryFree(subReq);
goto end;
}
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
@@ -1390,6 +1395,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
end:
taosMemoryFreeClear(pTableMeta);
qDestroyQuery(pQuery);
+ taosMemoryFree(subReq);
return code;
}
diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 24da5f7b70..5c37822222 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -299,6 +299,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
for (; i < taosArrayGetSize(cols); i++) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) {
+ taosHashCleanup(hashTmp);
return -1;
}
}
@@ -430,7 +431,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
STableMeta *pTableMeta = NULL;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
- strcpy(pName.dbname, info->pRequest->pDb);
+ tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
SRequestConnInfo conn = {0};
conn.pTrans = info->taos->pAppInfo->pTransporter;
@@ -874,7 +875,8 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra
kv->i = ts;
kv->type = TSDB_DATA_TYPE_TIMESTAMP;
kv->length = (int16_t)tDataTypes[kv->type].bytes;
- if (cols) taosArrayPush(cols, &kv);
+ taosArrayPush(cols, &kv);
+
return TSDB_CODE_SUCCESS;
}
@@ -1009,6 +1011,7 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t *
static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey,
SSmlMsgBuf *msg) {
+ if(!cols) return TSDB_CODE_OUT_OF_MEMORY;
const char *sql = data;
size_t childTableNameLen = strlen(tsSmlChildTableName);
while (*sql != '\0') {
@@ -1082,7 +1085,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab
kv->length = valueLen;
kv->type = TSDB_DATA_TYPE_NCHAR;
- if (cols) taosArrayPush(cols, &kv);
+ taosArrayPush(cols, &kv);
}
return TSDB_CODE_SUCCESS;
@@ -1370,8 +1373,14 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
SHashObj *s2 = *(SHashObj **)key2;
SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN);
SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN);
- ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP);
- ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP);
+ if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){
+ uError("smlKvTimeHashCompare kv1");
+ return -1;
+ }
+ if(!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP){
+ uError("smlKvTimeHashCompare kv2");
+ return -1;
+ }
if (kv1->i < kv2->i) {
return -1;
} else if (kv1->i > kv2->i) {
@@ -1735,7 +1744,7 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
kv->i = tsVal;
kv->type = TSDB_DATA_TYPE_TIMESTAMP;
kv->length = (int16_t)tDataTypes[kv->type].bytes;
- if (cols) taosArrayPush(cols, &kv);
+ taosArrayPush(cols, &kv);
return TSDB_CODE_SUCCESS;
}
@@ -1932,6 +1941,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
}
static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) {
+ if(!cols) return TSDB_CODE_OUT_OF_MEMORY;
cJSON *metricVal = cJSON_GetObjectItem(root, "value");
if (metricVal == NULL) {
return TSDB_CODE_TSC_INVALID_JSON;
@@ -1941,7 +1951,7 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) {
if (!kv) {
return TSDB_CODE_OUT_OF_MEMORY;
}
- if (cols) taosArrayPush(cols, &kv);
+ taosArrayPush(cols, &kv);
kv->key = VALUE;
kv->keyLen = VALUE_LEN;
@@ -1955,7 +1965,9 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) {
static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey,
SSmlMsgBuf *msg) {
int32_t ret = TSDB_CODE_SUCCESS;
-
+ if (!pKVs){
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
cJSON *tags = cJSON_GetObjectItem(root, "tags");
if (tags == NULL || tags->type != cJSON_Object) {
return TSDB_CODE_TSC_INVALID_JSON;
@@ -1985,14 +1997,14 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN
return TSDB_CODE_TSC_INVALID_JSON;
}
memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
- strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN);
+ tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN);
continue;
}
// add kv to SSmlKv
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
if (!kv) return TSDB_CODE_OUT_OF_MEMORY;
- if (pKVs) taosArrayPush(pKVs, &kv);
+ taosArrayPush(pKVs, &kv);
// key
kv->keyLen = keyLen;
@@ -2103,6 +2115,8 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) {
if (!oneTable) {
tinfo = smlBuildTableInfo();
if (!tinfo) {
+ smlDestroyCols(cols);
+ if (info->dataFormat) taosArrayDestroy(cols);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES);
@@ -2295,7 +2309,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
SSmlTableInfo *tableData = *oneTable;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
- strcpy(pName.dbname, info->pRequest->pDb);
+ tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName));
SRequestConnInfo conn = {0};
diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c
index 7a800bd334..bf3fd00f14 100644
--- a/source/client/src/clientStmt.c
+++ b/source/client/src/clientStmt.c
@@ -201,6 +201,9 @@ int32_t stmtCacheBlock(STscStmt* pStmt) {
}
STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
+ if(!pSrc){
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
STableDataBlocks* pDst = NULL;
STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc));
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 922cab23c8..8f94d88b1c 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -233,12 +233,12 @@ void tmq_conf_destroy(tmq_conf_t* conf) {
tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) {
if (strcmp(key, "group.id") == 0) {
- strcpy(conf->groupId, value);
+ tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN);
return TMQ_CONF_OK;
}
if (strcmp(key, "client.id") == 0) {
- strcpy(conf->clientId, value);
+ tstrncpy(conf->clientId, value, 256);
return TMQ_CONF_OK;
}
@@ -452,7 +452,6 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
int32_t code;
tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
if (code < 0) {
- ASSERT(0);
return -1;
}
void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
@@ -464,15 +463,22 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
SEncoder encoder;
tEncoderInit(&encoder, abuf, len);
tEncodeSTqOffset(&encoder, pOffset);
+ tEncoderClear(&encoder);
// build param
SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam));
+ if (pParam == NULL) {
+ taosMemoryFree(buf);
+ return -1;
+ }
pParam->params = pParamSet;
pParam->pOffset = pOffset;
// build send info
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (pMsgSendInfo == NULL) {
+ taosMemoryFree(buf);
+ taosMemoryFree(pParam);
return -1;
}
pMsgSendInfo->msgInfo = (SDataBuf){
@@ -547,6 +553,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm
if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) {
if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) {
+ tsem_destroy(&pParamSet->rspSem);
+ taosMemoryFree(pParamSet);
goto FAIL;
}
goto HANDLE_RSP;
@@ -565,6 +573,7 @@ HANDLE_RSP:
tsem_wait(&pParamSet->rspSem);
code = pParamSet->rspErr;
tsem_destroy(&pParamSet->rspSem);
+ taosMemoryFree(pParamSet);
return code;
} else {
code = 0;
@@ -587,7 +596,14 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
if (pParamSet == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ if (async) {
+ if (automatic) {
+ tmq->commitCb(tmq, code, tmq->commitCbUserParam);
+ } else {
+ userCb(tmq, code, userParam);
+ }
+ }
return -1;
}
@@ -642,16 +658,6 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t
code = pParamSet->rspErr;
tsem_destroy(&pParamSet->rspSem);
taosMemoryFree(pParamSet);
- } else {
- code = 0;
- }
-
- if (code != 0 && async) {
- if (automatic) {
- tmq->commitCb(tmq, code, tmq->commitCbUserParam);
- } else {
- userCb(tmq, code, userParam);
- }
}
#if 0
@@ -709,6 +715,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
int64_t refId = *(int64_t*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
+ taosMemoryFree(param);
return;
}
int64_t consumerId = tmq->consumerId;
@@ -721,6 +728,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
taosMemoryFree(pReq);
+ goto OVER;
}
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
@@ -870,8 +878,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t));
if (pTmq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
- tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(),
- pTmq->groupId);
+ tscError("setting up new consumer failed since %s, consumer group %s", terrstr(), conf->groupId);
return NULL;
}
@@ -939,10 +946,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
return NULL;
}
- int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
- *pRefId = pTmq->refId;
-
if (pTmq->hbBgEnable) {
+ int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
+ *pRefId = pTmq->refId;
pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer);
}
@@ -966,14 +972,14 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
SCMSubscribeReq req = {0};
int32_t code = -1;
+ tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz);
+
req.consumerId = tmq->consumerId;
tstrncpy(req.clientId, tmq->clientId, 256);
tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN);
req.topicNames = taosArrayInit(sz, sizeof(void*));
if (req.topicNames == NULL) goto FAIL;
- tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz);
-
for (int32_t i = 0; i < sz; i++) {
char* topic = taosArrayGetP(container, i);
@@ -1061,9 +1067,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
code = 0;
FAIL:
if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree);
- if (code != 0 && buf) {
- taosMemoryFree(buf);
- }
+ taosMemoryFree(buf);
+
return code;
}
@@ -1101,7 +1106,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM);
if (pRspWrapper == NULL) {
- taosMemoryFree(pMsg->pData);
tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch);
goto CREATE_MSG_FAIL;
}
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index ad5772e0fe..269e72f151 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -916,6 +916,10 @@ char *tTagValToData(const STagVal *value, bool isJson) {
}
bool tTagGet(const STag *pTag, STagVal *pTagVal) {
+ if(!pTag || !pTagVal){
+ return false;
+ }
+
int16_t lidx = 0;
int16_t ridx = pTag->nTag - 1;
int16_t midx;
diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
index a4c37dd334..17cbde437b 100644
--- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
+++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
@@ -41,8 +41,8 @@ typedef struct SMnodeMgmt {
} SMnodeMgmt;
// mmFile.c
-int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed);
-int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed);
+int32_t mmReadFile(const char *path, SMnodeOpt *pOption);
+int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption);
// mmHandle.c
SArray *mmGetMsgHandles();
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
index 27a35ae17a..c5ddb9f021 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
@@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "mmInt.h"
-int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
+int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
int32_t code = TSDB_CODE_INVALID_JSON_FORMAT;
int32_t len = 0;
int32_t maxLen = 4096;
@@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
char file[PATH_MAX] = {0};
TdFilePtr pFile = NULL;
- snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+ snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
code = 0;
@@ -50,38 +50,69 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
dError("failed to read %s since deployed not found", file);
goto _OVER;
}
- *pDeployed = deployed->valueint;
+ pOption->deploy = deployed->valueint;
- cJSON *id = cJSON_GetObjectItem(root, "id");
- if (id) {
- if (id->type != cJSON_Number) {
- dError("failed to read %s since id not found", file);
+ cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex");
+ if (selfIndex) {
+ if (selfIndex->type != cJSON_Number) {
+ dError("failed to read %s since selfIndex not found", file);
goto _OVER;
}
- if (pReplica) {
- pReplica->id = id->valueint;
- }
+ pOption->selfIndex = selfIndex->valueint;
}
- cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn");
- if (fqdn) {
- if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
- dError("failed to read %s since fqdn not found", file);
+ cJSON *replicas = cJSON_GetObjectItem(root, "replicas");
+ if (replicas) {
+ if (replicas->type != cJSON_Array) {
+ dError("failed to read %s since replicas not found", file);
goto _OVER;
}
- if (pReplica) {
- tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
- }
- }
- cJSON *port = cJSON_GetObjectItem(root, "port");
- if (port) {
- if (port->type != cJSON_Number) {
- dError("failed to read %s since port not found", file);
+ int32_t numOfReplicas = cJSON_GetArraySize(replicas);
+ if (numOfReplicas <= 0) {
+ dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas);
goto _OVER;
}
- if (pReplica) {
- pReplica->port = (uint16_t)port->valueint;
+ pOption->numOfReplicas = numOfReplicas;
+
+ for (int32_t i = 0; i < numOfReplicas; ++i) {
+ SReplica *pReplica = pOption->replicas + i;
+
+ cJSON *replica = cJSON_GetArrayItem(replicas, i);
+ if (replica == NULL) break;
+
+ cJSON *id = cJSON_GetObjectItem(replica, "id");
+ if (id) {
+ if (id->type != cJSON_Number) {
+ dError("failed to read %s since id not found", file);
+ goto _OVER;
+ }
+ if (pReplica) {
+ pReplica->id = id->valueint;
+ }
+ }
+
+ cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn");
+ if (fqdn) {
+ if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
+ dError("failed to read %s since fqdn not found", file);
+ goto _OVER;
+ }
+ if (pReplica) {
+ tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
+ }
+ }
+
+ cJSON *port = cJSON_GetObjectItem(replica, "port");
+ if (port) {
+ if (port->type != cJSON_Number) {
+ dError("failed to read %s since port not found", file);
+ goto _OVER;
+ }
+ if (pReplica) {
+ pReplica->port = (uint16_t)port->valueint;
+ }
+ }
}
}
@@ -92,18 +123,18 @@ _OVER:
if (root != NULL) cJSON_Delete(root);
if (pFile != NULL) taosCloseFile(&pFile);
if (code == 0) {
- dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+ dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy);
}
terrno = code;
return code;
}
-int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) {
+int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
- snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
- snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+ snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
+ snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
@@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
char *content = taosMemoryCalloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
- if (pReplica != NULL && pReplica->id > 0) {
- len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id);
- len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn);
- len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port);
+ if (pOption->deploy && pOption->numOfReplicas > 0) {
+ len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex);
+ len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n");
+
+ for (int32_t i = 0; i < pOption->numOfReplicas; ++i) {
+ const SReplica *pReplica = pOption->replicas + i;
+ if (pReplica != NULL && pReplica->id > 0) {
+ len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id);
+ len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn);
+ len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port);
+ }
+ if (i < pOption->numOfReplicas - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }],\n");
+ }
+ }
}
- len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed);
+ len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy);
len += snprintf(content + len, maxLen - len, "}\n");
taosWriteFile(pFile, content, len);
@@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
return -1;
}
- dDebug("successed to write %s, deployed:%d", realfile, deployed);
+ dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy);
return 0;
}
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
index ec761e6441..fc45dbf15f 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
return -1;
}
- if (createReq.replica != 1) {
+ SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1};
+ memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas));
+ for (int32_t i = 0; i < option.numOfReplicas; ++i) {
+ if (createReq.replicas[i].id == pInput->pData->dnodeId) {
+ option.selfIndex = i;
+ }
+ }
+
+ if (option.selfIndex == -1) {
terrno = TSDB_CODE_INVALID_OPTION;
- dGError("failed to create mnode since %s", terrstr());
+ dGError("failed to create mnode since %s, selfIndex is -1", terrstr());
return -1;
}
- bool deployed = true;
-
- SMnodeMgmt mgmt = {0};
- mgmt.path = pInput->path;
- mgmt.name = pInput->name;
- if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) {
+ if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
}
@@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
return -1;
}
- bool deployed = false;
-
- SMnodeMgmt mgmt = {0};
- mgmt.path = pInput->path;
- mgmt.name = pInput->name;
- if (mmWriteFile(&mgmt, NULL, deployed) != 0) {
+ SMnodeOpt option = {.deploy = false};
+ if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
}
@@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
- // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c
index 49207225a5..e7f71ad420 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c
@@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
}
static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
- SMnodeMgmt mgmt = {0};
- mgmt.path = pInput->path;
- if (mmReadFile(&mgmt, NULL, required) != 0) {
+ SMnodeOpt option = {0};
+ if (mmReadFile(pInput->path, &option) != 0) {
return -1;
}
- if (!(*required)) {
+ if (!option.deploy) {
*required = mmDeployRequired(pInput);
+ } else {
+ *required = true;
}
return 0;
}
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
- pOption->standby = false;
pOption->deploy = true;
pOption->msgCb = pMgmt->msgCb;
pOption->dnodeId = pMgmt->pData->dnodeId;
- pOption->replica.id = 1;
- pOption->replica.port = tsServerPort;
- tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
+ pOption->selfIndex = 0;
+ pOption->numOfReplicas = 1;
+ pOption->replicas[0].id = 1;
+ pOption->replicas[0].port = tsServerPort;
+ tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
}
-static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) {
- pOption->standby = false;
+static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
pOption->deploy = false;
pOption->msgCb = pMgmt->msgCb;
pOption->dnodeId = pMgmt->pData->dnodeId;
- if (pReplica->id > 0) {
- pOption->standby = true;
- pOption->replica = *pReplica;
- }
}
static void mmClose(SMnodeMgmt *pMgmt) {
@@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
- bool deployed = false;
- SReplica replica = {0};
- if (mmReadFile(pMgmt, &replica, &deployed) != 0) {
+ SMnodeOpt option = {0};
+ if (mmReadFile(pMgmt->path, &option) != 0) {
dError("failed to read file since %s", terrstr());
mmClose(pMgmt);
return -1;
}
- SMnodeOpt option = {0};
- if (!deployed) {
+ if (!option.deploy) {
dInfo("mnode start to deploy");
pMgmt->pData->dnodeId = 1;
mmBuildOptionForDeploy(pMgmt, pInput, &option);
} else {
dInfo("mnode start to open");
- mmBuildOptionForOpen(pMgmt, &replica, &option);
+ mmBuildOptionForOpen(pMgmt, &option);
}
pMgmt->pMnode = mndOpen(pMgmt->path, &option);
@@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
}
tmsgReportStartup("mnode-worker", "initialized");
- if (!deployed || replica.id > 0) {
- deployed = true;
- if (mmWriteFile(pMgmt, NULL, deployed) != 0) {
+ if (option.numOfReplicas > 0) {
+ option.deploy = true;
+ option.numOfReplicas = 0;
+ if (mmWriteFile(pMgmt->path, &option) != 0) {
dError("failed to write mnode file since %s", terrstr());
return -1;
}
diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
index d07ec7abb0..076826ebc2 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
@@ -51,26 +51,14 @@ static int32_t dmInitMonitor() {
static bool dmCheckDiskSpace() {
osUpdate();
- if (!osDataSpaceAvailable()) {
- dError("free disk size: %f GB, too little, require %f GB at least at least , quit",
- (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
- (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
- terrno = TSDB_CODE_NO_AVAIL_DISK;
- return false;
+ if (!osDataSpaceSufficient()) {
+ dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
}
- if (!osLogSpaceAvailable()) {
- dError("free disk size: %f GB, too little, require %f GB at least at least, quit",
- (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
- (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
- terrno = TSDB_CODE_NO_AVAIL_DISK;
- return false;
+ if (!osLogSpaceSufficient()) {
+ dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
}
- if (!osTempSpaceAvailable()) {
- dError("free disk size: %f GB, too little, require %f GB at least at least, quit",
- (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
- (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
- terrno = TSDB_CODE_NO_AVAIL_DISK;
- return false;
+ if (!osTempSpaceSufficient()) {
+ dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
}
return true;
}
diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h
index 71315a39fd..f55e830a44 100644
--- a/source/dnode/mnode/impl/inc/mndInt.h
+++ b/source/dnode/mnode/impl/inc/mndInt.h
@@ -87,12 +87,13 @@ typedef struct {
typedef struct {
tsem_t syncSem;
int64_t sync;
- SReplica replica;
int32_t errCode;
int32_t transId;
SRWLatch lock;
- int8_t standby;
int8_t leaderTransferFinish;
+ int8_t selfIndex;
+ int8_t numOfReplicas;
+ SReplica replicas[TSDB_MAX_REPLICA];
} SSyncMgmt;
typedef struct {
@@ -130,11 +131,10 @@ typedef struct SMnode {
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
int64_t mndGenerateUid(const char *name, int32_t len);
-int32_t mndAcquireRpcRef(SMnode *pMnode);
-void mndReleaseRpcRef(SMnode *pMnode);
-void mndSetRestore(SMnode *pMnode, bool restored);
-void mndSetStop(SMnode *pMnode);
-bool mndGetStop(SMnode *pMnode);
+void mndSetRestored(SMnode *pMnode, bool restored);
+bool mndGetRestored(SMnode *pMnode);
+void mndSetStop(SMnode *pMnode);
+bool mndGetStop(SMnode *pMnode);
#ifdef __cplusplus
}
diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h
index cb9d70d5ee..993efbcd08 100644
--- a/source/dnode/mnode/impl/inc/mndSync.h
+++ b/source/dnode/mnode/impl/inc/mndSync.h
@@ -24,7 +24,7 @@ extern "C" {
int32_t mndInitSync(SMnode *pMnode);
void mndCleanupSync(SMnode *pMnode);
-bool mndIsMaster(SMnode *pMnode);
+bool mndIsLeader(SMnode *pMnode);
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId);
void mndSyncStart(SMnode *pMnode);
void mndSyncStop(SMnode *pMnode);
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 6670211d78..3b31873857 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
}
while (numOfRows < rowsCapacity) {
- pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus);
+ pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true);
if (pShow->pIter == NULL) break;
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 9ade39e8e7..fba1fd94d6 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
goto _OVER;
} else {
pDnode->accessTimes++;
- mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
+ mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
}
}
@@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
}
pDnode->lastAccessTime = curMs;
+ pDnode->accessTimes++;
code = 0;
_OVER:
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index 275fc76e36..5fb23b045c 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -43,6 +43,37 @@
#include "mndUser.h"
#include "mndVgroup.h"
+static inline int32_t mndAcquireRpc(SMnode *pMnode) {
+ int32_t code = 0;
+ taosThreadRwlockRdlock(&pMnode->lock);
+ if (pMnode->stopped) {
+ terrno = TSDB_CODE_APP_NOT_READY;
+ code = -1;
+ } else if (!mndIsLeader(pMnode)) {
+ code = -1;
+ } else {
+#if 1
+ atomic_add_fetch_32(&pMnode->rpcRef, 1);
+#else
+ int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
+ mTrace("mnode rpc is acquired, ref:%d", ref);
+#endif
+ }
+ taosThreadRwlockUnlock(&pMnode->lock);
+ return code;
+}
+
+static inline void mndReleaseRpc(SMnode *pMnode) {
+ taosThreadRwlockRdlock(&pMnode->lock);
+#if 1
+ atomic_sub_fetch_32(&pMnode->rpcRef, 1);
+#else
+ int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
+ mTrace("mnode rpc is released, ref:%d", ref);
+#endif
+ taosThreadRwlockUnlock(&pMnode->lock);
+}
+
static void *mndBuildTimerMsg(int32_t *pContLen) {
SMTimerReq timerReq = {0};
@@ -201,7 +232,7 @@ static int32_t mndInitWal(SMnode *pMnode) {
pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) {
- mError("failed to open wal since %s", terrstr());
+ mError("failed to open wal since %s. wal:%s", terrstr(), path);
return -1;
}
@@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) {
static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
pMnode->msgCb = pOption->msgCb;
pMnode->selfDnodeId = pOption->dnodeId;
- pMnode->syncMgmt.replica = pOption->replica;
- pMnode->syncMgmt.standby = pOption->standby;
+ pMnode->syncMgmt.selfIndex = pOption->selfIndex;
+ pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas;
+ memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas));
}
SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
@@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) {
mError("failed to deploy sdb while start mnode");
return -1;
}
- mndSetRestore(pMnode, true);
+ mndSetRestored(pMnode, true);
}
grantReset(pMnode, TSDB_GRANT_ALL, 0);
@@ -570,23 +602,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) {
return 0;
}
- if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
+ if (mndAcquireRpc(pMsg->info.node) == 0) return 0;
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
pMsg->msgType == TDMT_MND_UPTIME_TIMER) {
return -1;
}
- SEpSet epSet = {0};
- mndGetMnodeEpSet(pMsg->info.node, &epSet);
+ SEpSet epSet = {0};
+ SMnode *pMnode = pMsg->info.node;
+ mndGetMnodeEpSet(pMnode, &epSet);
const STraceId *trace = &pMsg->info.traceId;
- mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(),
- TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
+ mDebug(
+ "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s "
+ "numOfEps:%d inUse:%d",
+ pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync),
+ syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
if (epSet.numOfEps > 0) {
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
- mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
+ mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
}
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
@@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
int32_t code = (*fp)(pMsg);
- mndReleaseRpcRef(pMnode);
+ mndReleaseRpc(pMnode);
if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
mGTrace("msg:%p, won't response immediately since in progress", pMsg);
@@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
- if (mndAcquireRpcRef(pMnode) != 0) return -1;
+ if (mndAcquireRpc(pMnode) != 0) return -1;
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
@@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) {
- mndReleaseRpcRef(pMnode);
+ mndReleaseRpc(pMnode);
return -1;
}
@@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pGrantInfo->timeseries_total = INT32_MAX;
}
- mndReleaseRpcRef(pMnode);
+ mndReleaseRpc(pMnode);
return 0;
}
@@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
return 0;
}
-int32_t mndAcquireRpcRef(SMnode *pMnode) {
- int32_t code = 0;
- taosThreadRwlockRdlock(&pMnode->lock);
- if (pMnode->stopped) {
- mTrace("mnode not running");
- terrno = TSDB_CODE_APP_NOT_READY;
- code = -1;
- } else if (!mndIsMaster(pMnode)) {
- mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored);
- code = -1;
- } else {
- int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
- // mTrace("mnode rpc is acquired, ref:%d", ref);
- }
- taosThreadRwlockUnlock(&pMnode->lock);
- return code;
-}
-
-void mndReleaseRpcRef(SMnode *pMnode) {
- taosThreadRwlockRdlock(&pMnode->lock);
- int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
- // mTrace("mnode rpc is released, ref:%d", ref);
- taosThreadRwlockUnlock(&pMnode->lock);
-}
-
-void mndSetRestore(SMnode *pMnode, bool restored) {
+void mndSetRestored(SMnode *pMnode, bool restored) {
if (restored) {
taosThreadRwlockWrlock(&pMnode->lock);
pMnode->restored = true;
diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c
index d0440359ff..c27241317c 100644
--- a/source/dnode/mnode/impl/src/mndMnode.c
+++ b/source/dnode/mnode/impl/src/mndMnode.c
@@ -36,6 +36,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq);
static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq);
static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter);
+static void mndReloadSyncConfig(SMnode *pMnode);
int32_t mndInitMnode(SMnode *pMnode) {
SSdbTable table = {
@@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
}
pObj->state = TAOS_SYNC_STATE_ERROR;
+ mndReloadSyncConfig(pSdb->pMnode);
return 0;
}
@@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) {
mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
pOld->updateTime = pNew->updateTime;
+ mndReloadSyncConfig(pSdb->pMnode);
+
return 0;
}
@@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (pIter == NULL) break;
if (pObj->id == pMnode->selfDnodeId) {
- if (mndIsMaster(pMnode)) {
+ if (mndIsLeader(pMnode)) {
pEpSet->inUse = pEpSet->numOfEps;
} else {
pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes;
@@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (pEpSet->numOfEps == 0) {
syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet);
}
+
+ if (pEpSet->inUse >= pEpSet->numOfEps) {
+ pEpSet->inUse = 0;
+ }
}
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
@@ -274,13 +282,72 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod
return 0;
}
+static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
+ int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
+ void *pReq = taosMemoryMalloc(contLen);
+ tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
+
+ STransAction action = {
+ .epSet = *pCreateEpSet,
+ .pCont = pReq,
+ .contLen = contLen,
+ .msgType = TDMT_DND_CREATE_MNODE,
+ .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED,
+ };
+
+ if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+ taosMemoryFree(pReq);
+ return -1;
+ }
+ return 0;
+}
+
+static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
+ int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
+ void *pReq = taosMemoryMalloc(contLen);
+ tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
+
+ STransAction action = {
+ .epSet = *pAlterEpSet,
+ .pCont = pReq,
+ .contLen = contLen,
+ .msgType = TDMT_MND_ALTER_MNODE,
+ .acceptableCode = 0,
+ };
+
+ if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+ taosMemoryFree(pReq);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
+ int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq);
+ void *pReq = taosMemoryMalloc(contLen);
+ tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq);
+
+ STransAction action = {
+ .epSet = *pDroprEpSet,
+ .pCont = pReq,
+ .contLen = contLen,
+ .msgType = TDMT_DND_DROP_MNODE,
+ .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
+ };
+
+ if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+ taosMemoryFree(pReq);
+ return -1;
+ }
+ return 0;
+}
+
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int32_t numOfReplicas = 0;
- SDAlterMnodeReq alterReq = {0};
SDCreateMnodeReq createReq = {0};
- SEpSet alterEpset = {0};
SEpSet createEpset = {0};
while (1) {
@@ -288,75 +355,25 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
- alterReq.replicas[numOfReplicas].id = pMObj->id;
- alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-
- alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
- if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
- alterEpset.inUse = numOfReplicas;
- }
+ createReq.replicas[numOfReplicas].id = pMObj->id;
+ createReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
+ memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
numOfReplicas++;
sdbRelease(pSdb, pMObj);
}
- alterReq.replica = numOfReplicas + 1;
- alterReq.replicas[numOfReplicas].id = pDnode->id;
- alterReq.replicas[numOfReplicas].port = pDnode->port;
- memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-
- alterEpset.numOfEps = numOfReplicas + 1;
- alterEpset.eps[numOfReplicas].port = pDnode->port;
- memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-
- createReq.replica = 1;
- createReq.replicas[0].id = pDnode->id;
- createReq.replicas[0].port = pDnode->port;
- memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
+ createReq.replica = numOfReplicas + 1;
+ createReq.replicas[numOfReplicas].id = pDnode->id;
+ createReq.replicas[numOfReplicas].port = pDnode->port;
+ memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
+ createEpset.inUse = 0;
createEpset.numOfEps = 1;
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
- {
- int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSDCreateMnodeReq(pReq, contLen, &createReq);
-
- STransAction action = {
- .epSet = createEpset,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_DND_CREATE_MNODE,
- .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
-
- {
- int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
-
- STransAction action = {
- .epSet = alterEpset,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_MND_ALTER_MNODE,
- .acceptableCode = 0,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
+ if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
return 0;
}
@@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
+ if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
- if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndTransAppendNullLog(pTrans) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -459,107 +476,28 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO
}
static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
- SSdb *pSdb = pMnode->pSdb;
- void *pIter = NULL;
- int32_t numOfReplicas = 0;
- SDAlterMnodeReq alterReq = {0};
- SDDropMnodeReq dropReq = {0};
- SSetStandbyReq standbyReq = {0};
- SEpSet alterEpset = {0};
- SEpSet dropEpSet = {0};
-
- while (1) {
- SMnodeObj *pMObj = NULL;
- pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
- if (pIter == NULL) break;
- if (pMObj->id == pObj->id) {
- sdbRelease(pSdb, pMObj);
- continue;
- }
-
- alterReq.replicas[numOfReplicas].id = pMObj->id;
- alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-
- alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
- if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
- alterEpset.inUse = numOfReplicas;
- }
-
- numOfReplicas++;
- sdbRelease(pSdb, pMObj);
- }
-
- alterReq.replica = numOfReplicas;
- alterEpset.numOfEps = numOfReplicas;
+ SSdb *pSdb = pMnode->pSdb;
+ void *pIter = NULL;
+ int32_t numOfReplicas = 0;
+ SDDropMnodeReq dropReq = {0};
+ SEpSet dropEpSet = {0};
dropReq.dnodeId = pDnode->id;
dropEpSet.numOfEps = 1;
dropEpSet.eps[0].port = pDnode->port;
memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
- standbyReq.dnodeId = pDnode->id;
- standbyReq.standby = 1;
-
- {
- int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq);
- SMsgHead *pHead = pReq;
- pHead->contLen = htonl(contLen);
- pHead->vgId = htonl(MNODE_HANDLE);
-
- STransAction action = {
- .epSet = dropEpSet,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_SYNC_SET_MNODE_STANDBY,
- .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
-
- {
- int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
-
- STransAction action = {
- .epSet = alterEpset,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_MND_ALTER_MNODE,
- .acceptableCode = 0,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
-
- {
- int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq);
-
- STransAction action = {
- .epSet = dropEpSet,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_DND_DROP_MNODE,
- .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
+ int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE);
+ if (totalMnodes == 2) {
+ mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
+ if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
+ if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+ } else if (totalMnodes == 3) {
+ mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
+ if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+ if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
+ } else {
+ return -1;
}
return 0;
@@ -567,7 +505,6 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
if (pObj == NULL) return 0;
- if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1;
if (mndTransAppendNullLog(pTrans) != 0) return -1;
@@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
int64_t curMs = taosGetTimestampMs();
while (numOfRows < rows) {
- pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus);
+ pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true);
if (pShow->pIter == NULL) break;
cols = 0;
@@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) {
}
static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
+#if 1
+ return 0;
+#else
SMnode *pMnode = pReq->info.node;
SDAlterMnodeReq alterReq = {0};
@@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
return -1;
}
+ SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
+ memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
+ for (int32_t i = 0; i < option.numOfReplicas; ++i) {
+ if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
+ option.selfIndex = i;
+ }
+ }
+
+ if (option.selfIndex == -1) {
+ mInfo("alter mnode not processed since selfIndex is -1", terrstr());
+ return 0;
+ }
+
+ if (mndWriteFile(pMnode->path, &option) != 0) {
+ mError("failed to write mnode file since %s", terrstr());
+ return -1;
+ }
+
SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
for (int32_t i = 0; i < alterReq.replica; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn));
pNode->nodePort = alterReq.replicas[i].port;
- if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i;
+ if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
+ cfg.myIndex = i;
+ }
}
if (cfg.myIndex == -1) {
mError("failed to alter mnode since myindex is -1");
return -1;
} else {
- mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex);
+ mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
for (int32_t i = 0; i < alterReq.replica; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
}
}
- mInfo("trans:-1, sync reconfig will be proposed");
-
- SSyncMgmt *pMgmt = &pMnode->syncMgmt;
- pMgmt->standby = 0;
- int32_t code = syncReconfig(pMgmt->sync, &cfg);
+ int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) {
- mError("trans:-1, failed to propose sync reconfig since %s", terrstr());
- return code;
+ mError("failed to sync reconfig since %s", terrstr());
} else {
- pMgmt->errCode = 0;
- taosWLockLatch(&pMgmt->lock);
- pMgmt->transId = -1;
- taosWUnLockLatch(&pMgmt->lock);
- tsem_wait(&pMgmt->syncSem);
- mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode));
- terrno = pMgmt->errCode;
- return pMgmt->errCode;
+ mInfo("alter mnode sync success");
+ }
+
+ return code;
+#endif
+}
+
+static void mndReloadSyncConfig(SMnode *pMnode) {
+ SSdb *pSdb = pMnode->pSdb;
+ SMnodeObj *pObj = NULL;
+ ESdbStatus objStatus = 0;
+ void *pIter = NULL;
+ bool hasUpdatingMnode = false;
+ SSyncCfg cfg = {.myIndex = -1};
+
+ while (1) {
+ pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false);
+ if (pIter == NULL) break;
+ if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) {
+ mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus));
+ hasUpdatingMnode = true;
+ }
+
+ if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) {
+ SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum];
+ tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn));
+ pNode->nodePort = pObj->pDnode->port;
+ if (pObj->pDnode->id == pMnode->selfDnodeId) {
+ cfg.myIndex = cfg.replicaNum;
+ }
+ cfg.replicaNum++;
+ }
+
+ sdbReleaseLock(pSdb, pObj, false);
+ }
+
+ if (cfg.myIndex == -1) {
+ mInfo("vgId:1, mnode not reload since selfIndex is -1");
+ return;
+ }
+
+ if (!mndGetRestored(pMnode)) {
+ mInfo("vgId:1, mnode not reload since restore not finished");
+ return;
+ }
+
+ if (hasUpdatingMnode) {
+ mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
+ for (int32_t i = 0; i < cfg.replicaNum; ++i) {
+ SNodeInfo *pNode = &cfg.nodeInfo[i];
+ mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
+ }
+
+ int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
+ if (code != 0) {
+ mError("vgId:1, failed to reconfig mnode sync since %s", terrstr());
+ } else {
+ mInfo("vgId:1, reconfig mnode sync success");
+ }
}
}
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 511a84290d..1c8ec0a302 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -416,7 +416,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
}
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
- if (pInnerTask == NULL) {
+ if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbRelease(pSdb, pVgroup);
qDestroyQueryPlan(pPlan);
diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c
index 0e4dd9fc3c..866a3fa8b9 100644
--- a/source/dnode/mnode/impl/src/mndSync.c
+++ b/source/dnode/mnode/impl/src/mndSync.c
@@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) {
if (!pMnode->deploy) {
mInfo("vgId:1, sync restore finished, and will handle outstanding transactions");
mndTransPullup(pMnode);
- mndSetRestore(pMnode, true);
+ mndSetRestored(pMnode, true);
} else {
mInfo("vgId:1, sync restore finished");
}
@@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) {
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP);
syncInfo.pWal = pMnode->pWal;
syncInfo.pFsm = mndSyncMakeFsm(pMnode);
- syncInfo.isStandBy = pMgmt->standby;
syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT;
- mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby);
- if (pMgmt->standby || pMgmt->replica.id > 0) {
- SSyncCfg *pCfg = &syncInfo.syncCfg;
- pCfg->replicaNum = 1;
- pCfg->myIndex = 0;
- SNodeInfo *pNode = &pCfg->nodeInfo[0];
- tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn));
- pNode->nodePort = pMgmt->replica.port;
- mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort);
+ mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas);
+ SSyncCfg *pCfg = &syncInfo.syncCfg;
+ pCfg->replicaNum = pMgmt->numOfReplicas;
+ pCfg->myIndex = pMgmt->selfIndex;
+ for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) {
+ SNodeInfo *pNode = &pCfg->nodeInfo[i];
+ tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
+ pNode->nodePort = pMgmt->replicas[i].port;
+ mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort);
}
tsem_init(&pMgmt->syncSem, 0, 0);
@@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) {
setPingTimerMS(pMgmt->sync, 5000);
setElectTimerMS(pMgmt->sync, 3000);
setHeartbeatTimerMS(pMgmt->sync, 500);
- /*
- setElectTimerMS(pMgmt->sync, 600);
- setHeartbeatTimerMS(pMgmt->sync, 300);
- */
mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0;
@@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
syncSetMsgCb(pMgmt->sync, &pMnode->msgCb);
syncStart(pMgmt->sync);
- mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby);
+ mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync);
}
void mndSyncStop(SMnode *pMnode) {
@@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) {
taosWUnLockLatch(&pMnode->syncMgmt.lock);
}
-bool mndIsMaster(SMnode *pMnode) {
+bool mndIsLeader(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
if (!syncIsReady(pMgmt->sync)) {
@@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) {
return false;
}
- if (!pMnode->restored) {
+ if (!mndGetRestored(pMnode)) {
terrno = TSDB_CODE_APP_NOT_READY;
return false;
}
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index d30cc8f9ca..a64b07fdbd 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -53,7 +53,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
-static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
+static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); }
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransTimer(SRpcMsg *pReq);
diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h
index f922baf329..be93bd2f59 100644
--- a/source/dnode/mnode/sdb/inc/sdb.h
+++ b/source/dnode/mnode/sdb/inc/sdb.h
@@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey);
* @param pObj The object of the row.
*/
void sdbRelease(SSdb *pSdb, void *pObj);
+void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock);
/**
* @brief Traverse a sdb table
@@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj);
* @return void* The next iterator of the table.
*/
void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj);
-void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status);
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock);
/**
* @brief Cancel a traversal
diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c
index ecdf8c71a7..0fee0fbca1 100644
--- a/source/dnode/mnode/sdb/src/sdbHash.c
+++ b/source/dnode/mnode/sdb/src/sdbHash.c
@@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) {
taosThreadRwlockUnlock(pLock);
}
-void sdbRelease(SSdb *pSdb, void *pObj) {
+void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) {
if (pObj == NULL) return;
SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow));
if (pRow->type >= SDB_MAX) return;
TdThreadRwlock *pLock = &pSdb->locks[pRow->type];
- taosThreadRwlockWrlock(pLock);
+ if (lock) {
+ taosThreadRwlockWrlock(pLock);
+ }
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "release");
@@ -342,9 +344,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
sdbFreeRow(pSdb, pRow, true);
}
- taosThreadRwlockUnlock(pLock);
+ if (lock) {
+ taosThreadRwlockUnlock(pLock);
+ }
}
+void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); }
+
void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
*ppObj = NULL;
@@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
return ppRow;
}
-void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) {
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) {
*ppObj = NULL;
SHashObj *hash = sdbGetHash(pSdb, type);
if (hash == NULL) return NULL;
TdThreadRwlock *pLock = &pSdb->locks[type];
- taosThreadRwlockRdlock(pLock);
+ if (lock) {
+ taosThreadRwlockRdlock(pLock);
+ }
SSdbRow **ppRow = taosHashIterate(hash, pIter);
while (ppRow != NULL) {
@@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat
*status = pRow->status;
break;
}
- taosThreadRwlockUnlock(pLock);
+ if (lock) {
+ taosThreadRwlockUnlock(pLock);
+ }
return ppRow;
}
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index f96afe6fba..69e6cdce9f 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -115,7 +115,6 @@ typedef struct {
typedef struct {
SMqDataRsp dataRsp;
- SMqRspHead rspHead;
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
SRpcHandleInfo pInfo;
} STqPushEntry;
@@ -183,6 +182,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore);
// tqSink
void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
+void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
// tqOffset
char* tqOffsetBuildFName(const char* path, int32_t ver);
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 84ce087e55..631ef09d4b 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -675,7 +675,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
SSchemaWrapper *pSchemaWrapper = &schema;
tDecoderInit(&dc, pData, nData);
- tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
+ (void)tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
tDecoderClear(&dc);
tdbFree(pData);
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index c8a4ff945d..2b7982d381 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema
return -1;
}
- strcpy(pMetaRsp->tbName, tbName);
+ strncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN);
pMetaRsp->numOfColumns = pSchema->nCols;
pMetaRsp->tableType = TSDB_NORMAL_TABLE;
pMetaRsp->sversion = pSchema->version;
@@ -693,6 +693,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
if (iCol >= pSchema->nCols) break;
pColumn = &pSchema->pSchema[iCol];
+ ASSERT(pAlterTbReq->colName);
if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break;
iCol++;
}
@@ -816,6 +817,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
const void *pData = NULL;
int nData = 0;
+ if (pAlterTbReq->tagName == NULL) {
+ terrno = TSDB_CODE_INVALID_MSG;
+ return -1;
+ }
+
// search name index
ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
if (ret < 0) {
@@ -938,6 +944,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
metaUpdateTagIdx(pMeta, &ctbEntry);
}
+ ASSERT(ctbEntry.ctbEntry.pTags);
SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid};
tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags,
((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
@@ -946,7 +953,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
tDecoderClear(&dc1);
tDecoderClear(&dc2);
- if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
+ taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
tdbTbcClose(pTbDbc);
@@ -1196,8 +1203,8 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
SMetaEntry stbEntry = {0};
STagIdxKey *pTagIdxKey = NULL;
int32_t nTagIdxKey;
- const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0];
- const void *pTagData = NULL; //
+ const SSchema *pTagColumn;
+ const void *pTagData = NULL;
int32_t nTagData = 0;
SDecoder dc = {0};
int32_t ret = 0;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index e1578b37f1..5e1cc15063 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -192,7 +192,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
return -1;
}
- memcpy(buf, &pPushEntry->rspHead, sizeof(SMqRspHead));
+ memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead));
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
@@ -215,7 +215,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
tFormatOffset(buf1, 80, &pRsp->reqOffset);
tFormatOffset(buf2, 80, &pRsp->rspOffset);
tqDebug("vgId:%d, from consumer:%" PRId64 ", (epoch %d) push rsp, block num: %d, reqOffset:%s, rspOffset:%s",
- TD_VID(pTq->pVnode), pPushEntry->rspHead.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2);
+ TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2);
return 0;
}
@@ -560,9 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN);
dataRsp.withTbName = 0;
memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp));
- pPushEntry->rspHead.consumerId = consumerId;
- pPushEntry->rspHead.epoch = reqEpoch;
- pPushEntry->rspHead.mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
+ pPushEntry->dataRsp.head.consumerId = consumerId;
+ pPushEntry->dataRsp.head.epoch = reqEpoch;
+ pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*));
tqDebug("tmq poll: consumer %ld, subkey %s, vg %d save handle to push mgr", consumerId, pHandle->subKey,
TD_VID(pTq->pVnode));
@@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) {
pTask->smaSink.smaSink = smaHandleRes;
} else if (pTask->outputType == TASK_OUTPUT__TABLE) {
pTask->tbSink.vnode = pTq->pVnode;
- pTask->tbSink.tbSinkFunc = tqTableSink;
+ pTask->tbSink.tbSinkFunc = tqTableSink1;
ASSERT(pTask->tbSink.pSchemaWrapper);
ASSERT(pTask->tbSink.pSchemaWrapper->pSchema);
@@ -1010,16 +1010,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
qError("stream task input del failed, task id %d", pTask->taskId);
+
+ taosFreeQitem(pRefBlock);
continue;
}
+
if (streamSchedExec(pTask) < 0) {
qError("stream task launch failed, task id %d", pTask->taskId);
continue;
}
+
} else {
streamTaskInputFail(pTask);
}
}
+
int32_t ref = atomic_sub_fetch_32(pRef, 1);
ASSERT(ref >= 0);
if (ref == 0) {
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index 521d12fdab..76bfa72caa 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -284,6 +284,212 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
return ret;
}
+void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
+ const SArray* pBlocks = (const SArray*)data;
+ SVnode* pVnode = (SVnode*)vnode;
+ int64_t suid = pTask->tbSink.stbUid;
+ char* stbFullName = pTask->tbSink.stbFullName;
+ STSchema* pTSchema = pTask->tbSink.pTSchema;
+ SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;
+
+ int32_t blockSz = taosArrayGetSize(pBlocks);
+ bool createTb = true;
+
+ SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
+ if (!tagArray) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return;
+ }
+
+ tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz);
+ for (int32_t i = 0; i < blockSz; i++) {
+ SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+ if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
+ SBatchDeleteReq deleteReq = {0};
+ deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq));
+ deleteReq.suid = suid;
+ tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq);
+
+ int32_t len;
+ int32_t code;
+ tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code);
+ if (code < 0) {
+ //
+ ASSERT(0);
+ }
+ SEncoder encoder;
+ void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
+ void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
+ tEncoderInit(&encoder, abuf, len);
+ tEncodeSBatchDeleteReq(&encoder, &deleteReq);
+ tEncoderClear(&encoder);
+ taosArrayDestroy(deleteReq.deleteReqs);
+
+ ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId;
+
+ SRpcMsg msg = {
+ .msgType = TDMT_VND_BATCH_DEL,
+ .pCont = serializedDeleteReq,
+ .contLen = len + sizeof(SMsgHead),
+ };
+ if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+ rpcFreeCont(serializedDeleteReq);
+ tqDebug("failed to put delete req into write-queue since %s", terrstr());
+ }
+ } else {
+ SVCreateTbReq createTbReq = {0};
+
+ // set const
+ createTbReq.flags = 0;
+ createTbReq.type = TSDB_CHILD_TABLE;
+ createTbReq.ctb.suid = suid;
+
+ // set super table name
+ SName name = {0};
+ tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+ createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName);
+
+ // set tag content
+ taosArrayClear(tagArray);
+ STagVal tagVal = {
+ .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
+ .type = TSDB_DATA_TYPE_UBIGINT,
+ .i64 = (int64_t)pDataBlock->info.groupId,
+ };
+ taosArrayPush(tagArray, &tagVal);
+ createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
+
+ STag* pTag = NULL;
+ tTagNew(tagArray, 1, false, &pTag);
+ if (pTag == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ taosArrayDestroy(tagArray);
+ tdDestroySVCreateTbReq(&createTbReq);
+ return;
+ }
+ createTbReq.ctb.pTag = (uint8_t*)pTag;
+
+ // set tag name
+ SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
+ char tagNameStr[TSDB_COL_NAME_LEN] = {0};
+ strcpy(tagNameStr, "group_id");
+ taosArrayPush(tagName, tagNameStr);
+ createTbReq.ctb.tagName = tagName;
+
+ // set table name
+ if (pDataBlock->info.parTbName[0]) {
+ createTbReq.name = strdup(pDataBlock->info.parTbName);
+ } else {
+ createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
+ }
+
+ int32_t schemaLen;
+ int32_t code;
+ tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
+ if (code < 0) {
+ tdDestroySVCreateTbReq(&createTbReq);
+ taosArrayDestroy(tagArray);
+ return;
+ }
+
+ // save schema str
+ void* schemaStr = taosMemoryMalloc(schemaLen);
+ if (schemaStr == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ tdDestroySVCreateTbReq(&createTbReq);
+ taosArrayDestroy(tagArray);
+ return;
+ }
+
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, schemaStr, schemaLen);
+ code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
+ if (code < 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ tdDestroySVCreateTbReq(&createTbReq);
+ taosArrayDestroy(tagArray);
+ tEncoderClear(&encoder);
+ taosMemoryFree(schemaStr);
+ return;
+ }
+ tEncoderClear(&encoder);
+ tdDestroySVCreateTbReq(&createTbReq);
+
+ int32_t cap = sizeof(SSubmitReq);
+
+ int32_t rows = pDataBlock->info.rows;
+ int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+
+ cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen;
+
+ SSubmitReq* ret = rpcMallocCont(cap);
+ ret->header.vgId = pVnode->config.vgId;
+ ret->length = sizeof(SSubmitReq);
+ ret->numOfBlocks = htonl(1);
+
+ SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq));
+
+ blkHead->numOfRows = htonl(pDataBlock->info.rows);
+ blkHead->sversion = htonl(pTSchema->version);
+ blkHead->suid = htobe64(suid);
+ // uid is assigned by vnode
+ blkHead->uid = 0;
+ blkHead->schemaLen = 0;
+
+ tqDebug("tq sink, convert block %d, rows: %d", i, rows);
+
+ int32_t dataLen = 0;
+ void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
+ STSRow* rowData = blkSchema;
+ if (createTb) {
+ memcpy(blkSchema, schemaStr, schemaLen);
+ blkHead->schemaLen = htonl(schemaLen);
+ rowData = POINTER_SHIFT(blkSchema, schemaLen);
+ }
+
+ taosMemoryFree(schemaStr);
+
+ for (int32_t j = 0; j < rows; j++) {
+ SRowBuilder rb = {0};
+ tdSRowInit(&rb, pTSchema->version);
+ tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen);
+ tdSRowResetBuf(&rb, rowData);
+
+ for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
+ const STColumn* pColumn = &pTSchema->columns[k];
+ SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k);
+ if (colDataIsNull_s(pColData, j)) {
+ tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k);
+ } else {
+ void* colData = colDataGetData(pColData, j);
+ tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k);
+ }
+ }
+ tdSRowEnd(&rb);
+ int32_t rowLen = TD_ROW_LEN(rowData);
+ rowData = POINTER_SHIFT(rowData, rowLen);
+ dataLen += rowLen;
+ }
+ blkHead->dataLen = htonl(dataLen);
+
+ ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen;
+ ret->length = htonl(ret->length);
+
+ SRpcMsg msg = {
+ .msgType = TDMT_VND_SUBMIT,
+ .pCont = ret,
+ .contLen = ntohl(ret->length),
+ };
+
+ if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+ rpcFreeCont(ret);
+ tqDebug("failed to put into write-queue since %s", terrstr());
+ }
+ }
+ }
+ taosArrayDestroy(tagArray);
+}
+
void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
const SArray* pRes = (const SArray*)data;
SVnode* pVnode = (SVnode*)vnode;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 50c7a7cc53..0ee2e31097 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD
if (code) goto _err;
}
+_err:
if (aDelData) {
taosArrayDestroy(aDelData);
}
-_err:
return code;
}
@@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
// code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
- if (code) goto _err;
*pDelIdx = *pIdx;
+_err:
if (pDelIdxArray) {
taosArrayDestroy(pDelIdxArray);
}
-_err:
return code;
}
@@ -599,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
* &state->blockIdx);
*/
state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ);
- if (code) goto _err;
-
if (!state->pBlockIdx) {
goto _next_fileset;
}
@@ -898,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
if (code) goto _err;
code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
- if (code) goto _err;
+ if (code) {
+ tsdbDelFReaderClose(&pDelFReader);
+ goto _err;
+ }
code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline);
- if (code) goto _err;
+ if (code) {
+ tsdbDelFReaderClose(&pDelFReader);
+ goto _err;
+ }
tsdbDelFReaderClose(&pDelFReader);
} else {
@@ -1231,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
_err:
nextRowIterClose(&iter);
taosMemoryFreeClear(pTSchema);
+ *ppLastArray = NULL;
+ taosArrayDestroy(pColArray);
return code;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index aca9042261..a4d993bc6c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -1045,7 +1045,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
- ASSERT(eno == 0);
+ ASSERT(eno == 0 &&
+ "tsdbCommit failure"
+ "Restart taosd");
code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
TSDB_CHECK_CODE(code, lino, _exit);
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c
index da410532f9..6fd5629592 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS.c
@@ -819,7 +819,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname);
- taosRemoveFile(fname);
+ (void)taosRemoveFile(fname);
taosMemoryFree(fSet.pHeadF);
}
} else {
@@ -961,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname);
- taosRemoveFile(fname);
+ (void)taosRemoveFile(fname);
taosMemoryFree(pSetOld->pHeadF);
}
@@ -1114,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
- taosRemoveFile(fname);
+ (void)taosRemoveFile(fname);
taosMemoryFree(pSet->pHeadF);
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index 450032d4f2..0189ced3c6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
goto _err;
}
+ ASSERT(pPool != NULL);
// do delete
SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData));
if (pDelData == NULL) {
@@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
+ ASSERT(pPool != NULL);
pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
if (pTbData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN
// node
level = tsdbMemSkipListRandLevel(&pTbData->sl);
+ ASSERT(pPool != NULL);
pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
if (pNode == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 09a46b59bb..2f4fdfc5f8 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -133,8 +133,15 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
id.uid = pIter->uid;
}
- tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
+ code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
+
code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
double el = (taosGetTimestampUs() - st) / 1000.0;
pInfo->elapsedTime += el;
@@ -144,9 +151,6 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s",
pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el,
idStr);
- if (code != TSDB_CODE_SUCCESS) {
- goto _exit;
- }
pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk;
tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr);
@@ -459,8 +463,8 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) {
}
bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
- int32_t code = 0;
int32_t step = pIter->backward ? -1 : 1;
+ terrno = TSDB_CODE_SUCCESS;
// no qualified last file block in current file, no need to fetch row
if (pIter->pSttBlk == NULL) {
@@ -469,6 +473,10 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
int32_t iBlockL = pIter->iSttBlk;
SBlockData *pBlockData = loadLastBlock(pIter, idStr);
+ if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
+
pIter->iRow += step;
while (1) {
@@ -494,11 +502,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow);
_exit:
- if (code != TSDB_CODE_SUCCESS) {
- terrno = code;
- }
-
- return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL);
+ return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL);
}
SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; }
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 6b706901ba..670691ab1f 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -340,7 +340,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader));
if (pIter->pLastBlockReader == NULL) {
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
- tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr);
+ tsdbError("failed to prepare the last block iterator, code:%s %s", tstrerror(code), pReader->idStr);
return code;
}
}
@@ -646,7 +646,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
double el = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug(
- "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
+ "load block of %"PRIzu" tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
"time:%.2f ms %s",
numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el,
pReader->idStr);
@@ -1515,6 +1515,11 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader*
taosMemoryFree(pReader->pMemSchema);
int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema);
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ return NULL;
+ }
+
return pReader->pMemSchema;
}
@@ -2676,9 +2681,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
int8_t level = 0;
int8_t precision = pVnode->config.tsdbCfg.precision;
int64_t now = taosGetTimestamp(precision);
- int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1
- : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000
- : 1000000);
+ int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L
+ : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
+ : 1000000L);
for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
SRetention* pRetention = retentions + level;
@@ -3085,7 +3090,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p
doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader);
tRowMerge(&merge, piRow);
- doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader);
+ doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader);
}
int32_t code = tRowMergerGetRow(&merge, pTSRow);
@@ -3372,14 +3377,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
if (pCond->suid != 0) {
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1);
if (pReader->pSchema == NULL) {
- tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1,
- pReader->idStr);
+ tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr);
}
} else if (taosArrayGetSize(pTableList) > 0) {
STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1);
if (pReader->pSchema == NULL) {
- tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr);
+ tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);
}
}
@@ -3444,7 +3448,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
return code;
_err:
- tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr);
+ tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
return code;
}
@@ -3733,7 +3737,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
}
int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
- if (isEmptyQueryTimeWindow(&pReader->window)) {
+ if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) {
return TSDB_CODE_SUCCESS;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index 911bd6b02c..84b3afe72c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -124,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
}
// check
- if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
+ if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
}
@@ -843,7 +843,8 @@ _err:
// SDataFReader ====================================================
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
int32_t code = 0;
- SDataFReader *pReader;
+ int32_t lino = 0;
+ SDataFReader *pReader = NULL;
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
char fname[TSDB_FILENAME_LEN];
@@ -851,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->pTsdb = pTsdb;
pReader->pSet = pSet;
@@ -859,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// head
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// data
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// sma
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// stt
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- *ppReader = pReader;
- return code;
+_exit:
+ if (code) {
+ *ppReader = NULL;
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
-_err:
- tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
- *ppReader = NULL;
+ if (pReader) {
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]);
+ tsdbCloseFile(&pReader->pSmaFD);
+ tsdbCloseFile(&pReader->pDataFD);
+ tsdbCloseFile(&pReader->pHeadFD);
+ taosMemoryFree(pReader);
+ }
+ } else {
+ *ppReader = pReader;
+ }
return code;
}
@@ -1588,4 +1598,4 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
_err:
tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
return code;
-}
\ No newline at end of file
+}
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 423ea2576e..74c704598d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -445,13 +445,14 @@ _err:
int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) {
int32_t code = 0;
+ int32_t lino = 0;
STsdbSnapReader* pReader = NULL;
// alloc
pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->pTsdb = pTsdb;
pReader->sver = sver;
@@ -461,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
code = taosThreadRwlockRdlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFSRef(pTsdb, &pReader->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = taosThreadRwlockUnlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
// data
@@ -485,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pIter->aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = tBlockDataCreate(&pIter->bData);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pReader->bData);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// del
pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
if (pReader->aDelIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
if (pReader->aDelData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
- *ppReader = pReader;
- return code;
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code),
+ pTsdb->path);
+ *ppReader = NULL;
-_err:
- tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
- tstrerror(code));
- *ppReader = NULL;
+ if (pReader) {
+ taosArrayDestroy(pReader->aDelData);
+ taosArrayDestroy(pReader->aDelIdx);
+ tBlockDataDestroy(&pReader->bData, 1);
+ tsdbFSDestroy(&pReader->fs);
+ taosMemoryFree(pReader);
+ }
+ } else {
+ *ppReader = pReader;
+ tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
+ }
return code;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c
index 730bd264a7..682f2b4225 100644
--- a/source/dnode/vnode/src/vnd/vnodeBufPool.c
+++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c
@@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) {
void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
SVBufPoolNode *pNode;
void *p = NULL;
+ ASSERT(pPool != NULL);
+
taosThreadSpinLock(&pPool->lock);
if (pPool->node.size >= pPool->ptr - pPool->node.data + size) {
// allocate from the anchor node
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 07d9b96261..07c4c32955 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) {
int vnodeShouldCommit(SVnode *pVnode) {
if (pVnode->inUse) {
- return pVnode->inUse->size > pVnode->inUse->node.size;
+ return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size);
}
return false;
}
@@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
data = NULL;
if (vnodeEncodeInfo(pInfo, &data) < 0) {
+ vError("failed to encode json info.");
return -1;
}
@@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
}
if (taosWriteFile(pFile, data, strlen(data)) < 0) {
- vError("failed to write info file:%s data:%s", fname, terrstr());
+ vError("failed to write info file:%s error:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
@@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) {
snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path);
}
if (vnodeSaveInfo(dir, &info) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
walBeginSnapshot(pVnode->pWal, pVnode->state.applied);
// preCommit
// smaSyncPreCommit(pVnode->pSma);
- if (smaAsyncPreCommit(pVnode->pSma) < 0) {
- ASSERT(0);
+ if(smaAsyncPreCommit(pVnode->pSma) < 0){
+ vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
@@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) {
// commit each sub-system
if (metaCommit(pVnode->pMeta) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (VND_IS_RSMA(pVnode)) {
if (smaAsyncCommit(pVnode->pSma) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbCommit(VND_RSMA0(pVnode)) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbCommit(VND_RSMA1(pVnode)) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbCommit(VND_RSMA2(pVnode)) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
} else {
if (tsdbCommit(pVnode->pTsdb) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
}
if (tqCommit(pVnode->pTq) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
// walCommit (TODO)
// commit info
if (vnodeCommitInfo(dir, &info) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
@@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) {
// postCommit
// smaSyncPostCommit(pVnode->pSma);
if (smaAsyncPostCommit(pVnode->pSma) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 298653d3ed..001bb5f7f2 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg));
if (pVnode->pWal == NULL) {
- vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno));
+ vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir);
goto _err;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index b83f790a3d..3556b47dd3 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -41,16 +41,28 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t nReqs;
tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
- tStartDecode(&dc);
+ if (tStartDecode(&dc) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
- tDecodeI32v(&dc, &nReqs);
+ if (tDecodeI32v(&dc, &nReqs) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ goto _err;
+ }
for (int32_t iReq = 0; iReq < nReqs; iReq++) {
tb_uid_t uid = tGenIdPI64();
char *name = NULL;
tStartDecode(&dc);
- tDecodeI32v(&dc, NULL);
- tDecodeCStr(&dc, &name);
+ if (tDecodeI32v(&dc, NULL) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
+ if (tDecodeCStr(&dc, &name) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
*(int64_t *)(dc.data + dc.pos) = uid;
*(int64_t *)(dc.data + dc.pos + 8) = ctime;
@@ -81,10 +93,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
char *name = NULL;
tDecoderInit(&dc, pBlock->data, msgIter.schemaLen);
- tStartDecode(&dc);
+ if (tStartDecode(&dc) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
- tDecodeI32v(&dc, NULL);
- tDecodeCStr(&dc, &name);
+ if (tDecodeI32v(&dc, NULL) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
+ if (tDecodeCStr(&dc, &name) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
uid = metaGetTableEntryUidByName(pVnode->pMeta, name);
if (uid == 0) {
@@ -148,6 +169,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
int32_t len;
int32_t ret;
+ if (!pVnode->inUse) {
+ terrno = TSDB_CODE_VND_NOT_SYNCED;
+ vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr());
+ return -1;
+ }
+
vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
version);
@@ -268,10 +295,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
_do_commit:
vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
// commit current change
- vnodeCommit(pVnode);
+ if (vnodeCommit(pVnode) < 0) {
+ vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
+ }
// start a new one
- vnodeBegin(pVnode);
+ if (vnodeBegin(pVnode) < 0) {
+ vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
+ }
}
return 0;
diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c
index 18923ffb67..4c4ba59fa9 100644
--- a/source/libs/executor/src/dataInserter.c
+++ b/source/libs/executor/src/dataInserter.c
@@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
}
pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows;
- qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows,
+ qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows,
pInserter->submitRes.affectedRows);
tFreeSSubmitRsp(pInserter->submitRes.pRsp);
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 0fadacb2c2..f99c832381 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1138,7 +1138,7 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i
s.bytes = bytes;
s.slotId = slotId;
s.precision = precision;
- strncpy(s.name, name, tListLen(s.name));
+ tstrncpy(s.name, name, tListLen(s.name));
return s;
}
@@ -1201,14 +1201,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) {
pExp->base.resSchema =
createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName);
- pExp->pExpr->_function.functionId = pFuncNode->funcId;
- pExp->pExpr->_function.pFunctNode = pFuncNode;
+ tExprNode* pExprNode = pExp->pExpr;
+
+ pExprNode->_function.functionId = pFuncNode->funcId;
+ pExprNode->_function.pFunctNode = pFuncNode;
+
+ tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName));
- strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
- tListLen(pExp->pExpr->_function.functionName));
#if 1
// todo refactor: add the parameter for tbname function
- if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
+ const char* name = "tbname";
+ int32_t len = strlen(name);
+
+ if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) &&
+ pExprNode->_function.functionName[len] == 0) {
+
pFuncNode->pParameterList = nodesMakeList();
ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
@@ -1359,7 +1366,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
} else {
char* udfName = pExpr->pExpr->_function.pFunctNode->functionName;
- strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN);
+ tstrncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN);
fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet);
}
pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index e527f7e7c1..41f861cd38 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -616,7 +616,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
int32_t nOptrWithVal = 0;
int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
- if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) {
+ if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) {
taosMemoryFreeClear(*pOutput);
*len = 0;
}
@@ -701,10 +701,10 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
return 0;
}
-int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo mtInfo) {
+int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) {
memset(pCond, 0, sizeof(SQueryTableDataCond));
pCond->order = TSDB_ORDER_ASC;
- pCond->numOfCols = mtInfo.schema->nCols;
+ pCond->numOfCols = pMtInfo->schema->nCols;
pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
if (pCond->colList == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
@@ -712,15 +712,15 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
}
pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
- pCond->suid = mtInfo.suid;
+ pCond->suid = pMtInfo->suid;
pCond->type = TIMEWINDOW_RANGE_CONTAINED;
pCond->startVersion = -1;
pCond->endVersion = sContext->snapVersion;
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
- pCond->colList[i].type = mtInfo.schema->pSchema[i].type;
- pCond->colList[i].bytes = mtInfo.schema->pSchema[i].bytes;
- pCond->colList[i].colId = mtInfo.schema->pSchema[i].colId;
+ pCond->colList[i].type = pMtInfo->schema->pSchema[i].type;
+ pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes;
+ pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId;
}
return TSDB_CODE_SUCCESS;
@@ -844,7 +844,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList);
if (mtInfo.uid == 0) return 0; // no data
- initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo);
+ initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0});
@@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
pTaskInfo->streamInfo.schema = mtInfo.schema;
- qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts);
+ qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts);
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
SStreamRawScanInfo* pInfo = pOperator->info;
SSnapContext* sContext = pInfo->sContext;
if (setForSnapShot(sContext, pOffset->uid) != 0) {
- qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid);
+ qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
return -1;
}
- qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid);
+ qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts);
} else if (pOffset->type == TMQ_OFFSET__LOG) {
SStreamRawScanInfo* pInfo = pOperator->info;
tsdbReaderClose(pInfo->dataReader);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index ea01b961b1..aaef411d2d 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -895,33 +895,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI
}
#endif
-static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
-#if 0
- SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
- uint32_t status = BLK_DATA_NOT_LOAD;
-
- int32_t numOfOutput = 0; // pTableScanInfo->numOfOutput;
- for (int32_t i = 0; i < numOfOutput; ++i) {
- int32_t functionId = pCtx[i].functionId;
- int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId;
-
- // group by + first/last should not apply the first/last block filter
- if (functionId < 0) {
- status |= BLK_DATA_DATA_LOAD;
- return status;
- } else {
- // status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId);
- // if ((status & BLK_DATA_DATA_LOAD) == BLK_DATA_DATA_LOAD) {
- // return status;
- // }
- }
- }
-
- return status;
-#endif
- return 0;
-}
-
int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) {
*status = BLK_DATA_NOT_LOAD;
@@ -1802,7 +1775,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
} else {
taosMemoryFree(pMsg->pData);
pSourceDataInfo->code = code;
- qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code));
+ qDebug("%s fetch rsp received, index:%d, code:%s", pSourceDataInfo->taskId, index, tstrerror(code));
}
pSourceDataInfo->status = EX_SOURCE_DATA_READY;
@@ -1857,6 +1830,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq));
if (NULL == pMsg) {
pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+ taosMemoryFree(pWrapper);
return pTaskInfo->code;
}
@@ -2815,6 +2789,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId);
blockDataCleanup(pInfo->pRes);
+ blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
+ blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
doApplyScalarCalculation(pOperator, pBlock, order, scanFlag);
if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) {
@@ -3396,8 +3372,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) {
SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++];
pSchema->colId = pColNode->colId;
pSchema->type = pColNode->node.resType.type;
- pSchema->type = pColNode->node.resType.bytes;
- strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
+ pSchema->bytes = pColNode->node.resType.bytes;
+ tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
}
// this the tags and pseudo function columns, we only keep the tag columns
@@ -3411,7 +3387,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) {
SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++];
pSchema->colId = pColNode->colId;
pSchema->type = pColNode->node.resType.type;
- pSchema->type = pColNode->node.resType.bytes;
+ pSchema->bytes = pColNode->node.resType.bytes;
strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
}
}
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index c3954d639f..15a7abf1ff 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -284,6 +284,7 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
return true;
}
+
static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
SExprSupp* pSup = &pTableScanInfo->pseudoSup;
@@ -324,19 +325,18 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
- pCost->skipBlocks += 1;
- doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
+ doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
+ pCost->skipBlocks += 1;
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
pCost->loadBlockStatis += 1;
loadSMA = true; // mark the operation of load sma;
bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo);
if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead
- doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
-
qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+ doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
return TSDB_CODE_SUCCESS;
} else {
qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo));
@@ -386,8 +386,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
}
relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true);
-
- // currently only the tbname pseudo column
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
if (pTableScanInfo->pFilterNode != NULL) {
@@ -1086,12 +1084,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) {
- SResultRowInfo dumyInfo;
+ SResultRowInfo dumyInfo = {0};
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
STimeWindow endWin = win;
STimeWindow preWin = win;
uint64_t groupId = gpIdCol[*pRowIndex];
+
while (1) {
if (hasGroup) {
(*pRowIndex) += 1;
@@ -1155,6 +1154,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
pResult->info.rows++;
}
}
+
+ blockDataDestroy(tmpBlock);
+
if (pResult->info.rows > 0) {
pResult->info.calWin = pInfo->updateWin;
return pResult;
@@ -1327,6 +1329,11 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock*
ASSERT(pResBlock->info.rows == 1);
// build tagArray
+ /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/
+ /*STagVal tagVal = {*/
+ /*.cid = 0,*/
+ /*.type = 0,*/
+ /*};*/
// build STag
// set STag
@@ -1545,7 +1552,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
tsdbReaderClose(pTSInfo->dataReader);
pTSInfo->dataReader = NULL;
tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
- qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1);
+ qDebug("queue scan tsdb over, switch to wal ver %"PRId64, pTaskInfo->streamInfo.snapshotVer + 1);
if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
return NULL;
}
@@ -2176,6 +2183,19 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
}
}
+ if (pTableScanNode->pTags != NULL) {
+ int32_t numOfTags;
+ SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags);
+ if (pTagExpr == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _error;
+ }
+ if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _error;
+ }
+ }
+
pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES);
if (pInfo->pBlockLists == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -2554,16 +2574,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
SMetaReader smrChildTable = {0};
metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
- metaGetTableEntryByName(&smrChildTable, condTableName);
+ int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
+ if (code != TSDB_CODE_SUCCESS) {
+ // terrno has been set by metaGetTableEntryByName, therefore, return directly
+ return NULL;
+ }
+
if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
metaReaderClear(&smrChildTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
+
SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK);
- metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+ code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+ if (code != TSDB_CODE_SUCCESS) {
+ // terrno has been set by metaGetTableEntryByUid
+ return NULL;
+ }
+
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smrSuperTable);
metaReaderClear(&smrChildTable);
@@ -2970,7 +3001,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
while (1) {
int64_t startTs = taosGetTimestampUs();
- strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
+ tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
strcpy(pInfo->req.user, pInfo->pUser);
int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req);
@@ -3781,6 +3812,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
+ taosArrayDestroy(pColList);
goto _error;
}
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index c651529c7a..0d5f521034 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataDestroy(p);
- qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
+ qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
pDataBlock->info.rows);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}
diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index 88af03775c..8b716531e5 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
&numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
pInfo->pCondition = pPhyFillNode->node.pConditions;
pInfo->pColMatchColInfo = pColMatchColInfo;
- initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+ int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
pInfo->srcRowIndex = 0;
pOperator->name = "FillOperator";
@@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo,
NULL, NULL, NULL);
- int32_t code = appendDownstream(pOperator, &downstream, 1);
+ code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
_error:
destroyStreamFillOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
+ pTaskInfo->code = code;
return NULL;
}
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index c1478bc935..cbbd874bb9 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* pGpDatas = (uint64_t*)pGpCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
- SResultRowInfo dumyInfo;
+ SResultRowInfo dumyInfo = {0};
dumyInfo.cur.pageId = -1;
+
STimeWindow win = {0};
if (IS_FINAL_OP(pInfo)) {
win.skey = startTsCols[i];
@@ -2841,14 +2842,13 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc);
+ initBasicInfo(&pInfo->binfo, pResBlock);
int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
- initBasicInfo(&pInfo->binfo, pResBlock);
-
pInfo->twAggSup.waterMark = pSessionNode->window.watermark;
pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType;
pInfo->gap = pSessionNode->gap;
@@ -4665,7 +4665,6 @@ void destroyStreamStateOperatorInfo(void* param) {
SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
destroyStreamSessionAggOperatorInfo(pChInfo);
taosMemoryFreeClear(pChild);
- taosMemoryFreeClear(pChInfo);
}
}
colDataDestroy(&pInfo->twAggSup.timeWindowData);
@@ -5828,27 +5827,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
pInfo->isFinal = false;
- if (pIntervalPhyNode->window.pExprs != NULL) {
- int32_t numOfScalar = 0;
- SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
- int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
- }
+
pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
initResultSizeInfo(&pOperator->resultInfo, 4096);
SExprSupp* pSup = &pOperator->exprSupp;
+
+ initBasicInfo(&pInfo->binfo, pResBlock);
+ initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
+ initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
+
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
- initBasicInfo(&pInfo->binfo, pResBlock);
- initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
- initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
+ if (pIntervalPhyNode->window.pExprs != NULL) {
+ int32_t numOfScalar = 0;
+ SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
+ code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+ }
pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
pInfo->invertible = false;
diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c
index 16fd11f97d..134862d88c 100644
--- a/source/libs/executor/src/tsimplehash.c
+++ b/source/libs/executor/src/tsimplehash.c
@@ -18,7 +18,7 @@
#include "tlog.h"
#define SHASH_DEFAULT_LOAD_FACTOR 0.75
-#define HASH_MAX_CAPACITY (1024 * 1024 * 16)
+#define HASH_MAX_CAPACITY (1024 * 1024 * 16L)
#define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
#define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl))
@@ -104,7 +104,7 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) {
int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
if (newCapacity > HASH_MAX_CAPACITY) {
- uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached",
+ uDebug("current capacity:%"PRIzu", maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached",
pHashObj->capacity, HASH_MAX_CAPACITY);
return;
}
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 0d29397471..7077a9b780 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -6169,6 +6169,99 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return pResInfo->numOfRes;
}
+int32_t interpFunction(SqlFunctionCtx* pCtx) {
+#if 0
+ int32_t fillType = (int32_t) pCtx->param[2].i64;
+ //bool ascQuery = (pCtx->order == TSDB_ORDER_ASC);
+
+ if (pCtx->start.key == pCtx->startTs) {
+ assert(pCtx->start.key != INT64_MIN);
+
+ COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->start.val);
+
+ goto interp_success_exit;
+ } else if (pCtx->end.key == pCtx->startTs && pCtx->end.key != INT64_MIN && fillType == TSDB_FILL_NEXT) {
+ COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->end.val);
+
+ goto interp_success_exit;
+ }
+
+ switch (fillType) {
+ case TSDB_FILL_NULL:
+ setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+ break;
+
+ case TSDB_FILL_SET_VALUE:
+ tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true);
+ break;
+
+ case TSDB_FILL_LINEAR:
+ if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs
+ || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
+ goto interp_exit;
+ }
+
+ double v1 = -1, v2 = -1;
+ GET_TYPED_DATA(v1, double, pCtx->inputType, &pCtx->start.val);
+ GET_TYPED_DATA(v2, double, pCtx->inputType, &pCtx->end.val);
+
+ SPoint point1 = {.key = pCtx->start.key, .val = &v1};
+ SPoint point2 = {.key = pCtx->end.key, .val = &v2};
+ SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput};
+
+ int32_t srcType = pCtx->inputType;
+ if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) {
+ setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+ } else {
+ bool exceedMax = false, exceedMin = false;
+ taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE, &exceedMax, &exceedMin);
+ if (exceedMax || exceedMin) {
+ __compar_fn_t func = getComparFunc((int32_t)pCtx->inputType, 0);
+ if (func(&pCtx->start.val, &pCtx->end.val) <= 0) {
+ COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->start.val : &pCtx->end.val);
+ } else {
+ COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, exceedMax ? &pCtx->end.val : &pCtx->start.val);
+ }
+ }
+ }
+ break;
+
+ case TSDB_FILL_PREV:
+ if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs) {
+ goto interp_exit;
+ }
+
+ COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->start.val);
+ break;
+
+ case TSDB_FILL_NEXT:
+ if (pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) {
+ goto interp_exit;
+ }
+
+ COPY_TYPED_DATA(pCtx->pOutput, pCtx->inputType, &pCtx->end.val);
+ break;
+
+ case TSDB_FILL_NONE:
+ // do nothing
+ default:
+ goto interp_exit;
+ }
+
+
+ interp_success_exit:
+ *(TSKEY*)pCtx->ptsOutputBuf = pCtx->startTs;
+ INC_INIT_VAL(pCtx, 1);
+
+ interp_exit:
+ pCtx->start.key = INT64_MIN;
+ pCtx->end.key = INT64_MIN;
+ pCtx->endTs = pCtx->startTs;
+#endif
+
+ return TSDB_CODE_SUCCESS;
+}
+
int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElems = 0;
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index dbe0c97ab6..4739a852c0 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -2357,7 +2357,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo
}
static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSelect) {
- if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) {
+ if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) ||
+ TSDB_SYSTEM_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType) {
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 0afc373f2d..101e99afea 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -169,7 +169,7 @@ typedef struct SSyncNode {
} SSyncNode;
// open/close --------------
-SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
+SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo);
void syncNodeStart(SSyncNode* pSyncNode);
void syncNodeStartStandBy(SSyncNode* pSyncNode);
void syncNodeClose(SSyncNode* pSyncNode);
diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c
index e000ba8bf8..4a35a15d3e 100644
--- a/source/libs/sync/src/syncAppendEntries.c
+++ b/source/libs/sync/src/syncAppendEntries.c
@@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset);
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset);
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c
index b604d25816..511113352e 100644
--- a/source/libs/sync/src/syncCommit.c
+++ b/source/libs/sync/src/syncCommit.c
@@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
} else {
pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index);
- ASSERT(pEntry != NULL);
+ if (pEntry == NULL) {
+ sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index);
+ return;
+ }
}
// cannot commit, even if quorum agree. need check term!
if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) {
@@ -127,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
// execute fsm
if (pSyncNode->pFsm != NULL) {
int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state);
- ASSERT(code == 0);
+ if (code != 0) {
+ wError("failed to commit sync node since %s", tstrerror(terrno));
+ }
}
}
}
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 378ac743b3..907eb21f4c 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -86,10 +86,10 @@ void syncCleanUp() {
}
}
-int64_t syncOpen(const SSyncInfo* pSyncInfo) {
+int64_t syncOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
if (pSyncNode == NULL) {
- sError("failed to open sync node. vgId:%d", pSyncInfo->vgId);
+ sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr());
return -1;
}
@@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg
return ret;
}
-int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
+int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) {
SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
if (pSyncNode == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
@@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
}
ASSERT(rid == pSyncNode->rid);
+#if 0
if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR;
@@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
return ret;
+#else
+ syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg);
+ syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID);
+ taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+ return 0;
+#endif
}
int32_t syncLeaderTransfer(int64_t rid) {
@@ -919,9 +926,7 @@ _END:
}
// open/close --------------
-SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
- SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo;
-
+SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode));
if (pSyncNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
sError("failed to open raft cfg file. path:%s", pSyncNode->configPath);
goto _error;
}
- pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
+ if (pSyncInfo->syncCfg.replicaNum == 0) {
+ pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
+ }
raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
@@ -2650,7 +2657,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
if (ths->state == TAOS_SYNC_STATE_LEADER) {
int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
- ASSERT(code == 0);
+ if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
+ return -1;
+ }
syncNodeReplicate(ths, false);
}
@@ -2726,7 +2736,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
if (code != 0) {
// del resp mgr, call FpCommitCb
- ASSERT(0);
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -2790,8 +2800,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
// del resp mgr, call FpCommitCb
- ASSERT(0);
return -1;
}
@@ -3043,7 +3053,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex,
pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
} else {
code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry);
- ASSERT(code == 0);
+ if (code != 0) {
+ sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i);
+ return -1;
+ }
ASSERT(pEntry != NULL);
}
diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c
index 496c8419de..c3dad104d1 100644
--- a/source/libs/sync/src/syncRaftLog.c
+++ b/source/libs/sync/src/syncRaftLog.c
@@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pEntry->index, err, err, errStr, sysErr, sysErrStr);
syncNodeErrorLog(pData->pSyncNode, logBuf);
-
- ASSERT(0);
return -1;
}
pEntry->index = index;
@@ -414,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
return 0;
}
-SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
+SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
- taosThreadMutexLock(&(pData->mutex));
-
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReader* pWalHandle = pData->pWalHandle;
ASSERT(pWalHandle != NULL);
@@ -444,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
}
} while (0);
- ASSERT(0);
+ sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index);
+ return NULL;
}
SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
@@ -465,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
terrno = saveErr;
*/
- taosThreadMutexUnlock(&(pData->mutex));
return pEntry;
} else {
@@ -473,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
}
}
+SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
+ SSyncLogStoreData* pData = pLogStore->data;
+ SSyncRaftEntry *pEntry = NULL;
+
+ taosThreadMutexLock(&pData->mutex);
+ pEntry = logStoreGetEntryWithoutLock(pLogStore, index);
+ taosThreadMutexUnlock(&pData->mutex);
+ return pEntry;
+}
+
int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index a148c1e36b..9e17f50dce 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
ret = tdbPagerWrite(pPager, pPage);
if (ret < 0) {
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN
ret = tdbPagerWrite(pPager, pChild);
if (ret < 0) {
- // TODO
- ASSERT(0);
- return 0;
+ tdbError("failed to write page since %s", terrstr());
+ return -1;
}
// Copy the root page content to the child page
@@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
ret = tdbPagerWrite(pBt->pPager, pOlds[i]);
if (ret < 0) {
- // TODO
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
}
@@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
ret = tdbPagerWrite(pBt->pPager, pParent);
if (ret < 0) {
- // TODO
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
ret = tdbPagerWrite(pBt->pPager, pNews[iNew]);
if (ret < 0) {
- // TODO
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
}
@@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt)
// mark dirty
ret = tdbPagerWrite(pBt->pPager, *ppOfp);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -987,7 +984,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
// fetch a new ofp and make it dirty
SPgno pgno = 0;
- SPage *ofp, *nextOfp;
+ SPage *ofp = NULL, *nextOfp = NULL;
ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt);
if (ret < 0) {
@@ -1009,6 +1006,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
nLeft -= kLen;
// pack partial val to local if any space left
if (nLocal > nHeader + kLen + sizeof(SPgno)) {
+ ASSERT(pVal != NULL && vLen != 0);
memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno));
nLeft -= nLocal - nHeader - kLen - sizeof(SPgno);
}
@@ -1942,7 +1940,7 @@ int tdbBtcDelete(SBTC *pBtc) {
// drop the cell on the leaf
ret = tdbPagerWrite(pPager, pBtc->pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -1964,7 +1962,7 @@ int tdbBtcDelete(SBTC *pBtc) {
if (idx < nCells) {
ret = tdbPagerWrite(pPager, pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -2029,7 +2027,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int
// mark dirty
ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index ea16e80562..6c01348bc2 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) {
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerBegin(pPager, pTxn);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
return -1;
}
}
@@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) {
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerCommit(pPager, pTxn);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
return -1;
}
}
@@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) {
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerAbort(pPager, pTxn);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
return -1;
}
}
diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c
index b6d1f95c0e..c73ddce74c 100644
--- a/source/libs/tdb/src/db/tdbPCache.c
+++ b/source/libs/tdb/src/db/tdbPCache.c
@@ -105,6 +105,7 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) {
if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) {
// TODO: handle error
+ tdbOsFree(pCache->aPage);
return -1;
}
@@ -267,7 +268,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
// 4. Try a create new page
if (!pPage) {
ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg);
- if (ret < 0) {
+ if (ret < 0 && pPage != NULL) {
// TODO
ASSERT(0);
return NULL;
@@ -300,8 +301,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
pPage->pPager = pPageH->pPager;
memcpy(pPage->pData, pPageH->pData, pPage->pageSize);
- tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage,
- TDB_PAGE_PGNO(pPageH));
+ // tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage,
+ // TDB_PAGE_PGNO(pPageH));
tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize);
pPage->kLen = pPageH->kLen;
pPage->vLen = pPageH->vLen;
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 58be8642d7..57ad2a783a 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -84,7 +84,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
pPager->pCache = pCache;
pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
- if (pPager->fd < 0) {
+ if (TDB_FD_INVALID(pPager->fd)) {
+ // if (pPager->fd < 0) {
return -1;
}
@@ -156,6 +157,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) {
ret = tdbPagerWrite(pPager, pPage);
if (ret < 0) {
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -210,7 +212,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
ret = tdbPagerWritePageToJournal(pPager, pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page to journal since %s", tstrerror(terrno));
return -1;
}
}
@@ -225,7 +227,9 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
// Open the journal
pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
- if (pPager->jfd < 0) {
+ if (TDB_FD_INVALID(pPager->jfd)) {
+ tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -243,9 +247,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
// sync the journal file
ret = tdbOsFSync(pPager->jfd);
if (ret < 0) {
- // TODO
- ASSERT(0);
- return 0;
+ tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
}
// loop to write the dirty pages to file
@@ -255,7 +259,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
pPage = (SPage *)pNode;
ret = tdbPagerWritePageToDB(pPager, pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1;
}
}
@@ -277,11 +281,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// sync the db file
- tdbOsFSync(pPager->fd);
+ if (tdbOsFSync(pPager->fd) < 0) {
+ tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
// remove the journal file
- tdbOsClose(pPager->jfd);
- tdbOsRemove(pPager->jFileName);
+ if (tdbOsClose(pPager->jfd) < 0) {
+ tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+ tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
pPager->inTran = 0;
return 0;
@@ -297,14 +315,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
// 0, sync the journal file
ret = tdbOsFSync(pPager->jfd);
if (ret < 0) {
- // TODO
- ASSERT(0);
- return 0;
+ tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
}
tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755);
if (jfd == NULL) {
- return 0;
+ return -1;
}
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
@@ -348,7 +366,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
// 4, remove the journal file
tdbOsClose(pPager->jfd);
- tdbOsRemove(pPager->jFileName);
+ (void)tdbOsRemove(pPager->jFileName);
pPager->inTran = 0;
return 0;
@@ -516,11 +534,16 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno));
if (ret < 0) {
+ tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize);
if (ret < 0) {
+ tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName,
+ pPage->pageSize);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -540,13 +563,16 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
- ASSERT(0);
+ tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName,
+ pPage->pageSize);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -580,33 +606,54 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
if (ret < 0) {
+ tdbOsFree(pageBuf);
return -1;
}
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) {
+ tdbOsFree(pageBuf);
return -1;
}
i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
- ASSERT(0);
+ tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ tdbOsFree(pageBuf);
return -1;
}
ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
+ pPager->pageSize);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ tdbOsFree(pageBuf);
return -1;
}
}
- tdbOsFSync(pPager->fd);
+ if (tdbOsFSync(pPager->fd) < 0) {
+ tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ tdbOsFree(pageBuf);
+ return -1;
+ }
tdbOsFree(pageBuf);
- tdbOsClose(jfd);
- tdbOsRemove(pPager->jFileName);
+ if (tdbOsClose(jfd) < 0) {
+ tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+ tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
return 0;
}
diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c
index 380baefb46..008907ca77 100644
--- a/source/libs/tdb/src/db/tdbTable.c
+++ b/source/libs/tdb/src/db/tdbTable.c
@@ -106,11 +106,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
// pTb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt));
if (ret < 0) {
+ tdbOsFree(pTb);
return -1;
}
ret = tdbPagerRestore(pPager, pTb->pBt);
if (ret < 0) {
+ tdbOsFree(pTb);
return -1;
}
diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h
index 503e109adb..b5dd27052c 100644
--- a/source/libs/tdb/src/inc/tdbOs.h
+++ b/source/libs/tdb/src/inc/tdbOs.h
@@ -37,6 +37,8 @@ extern "C" {
/* file */
typedef TdFilePtr tdb_fd_t;
+#define TDB_FD_INVALID(fd) (fd == NULL)
+
#define TDB_O_CREAT TD_FILE_CREATE
#define TDB_O_WRITE TD_FILE_WRITE
#define TDB_O_READ TD_FILE_READ
@@ -141,4 +143,4 @@ typedef pthread_mutex_t tdb_mutex_t;
}
#endif
-#endif /*_TDB_OS_H_*/
\ No newline at end of file
+#endif /*_TDB_OS_H_*/
diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index 144cce2cd0..279f4dc656 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -35,113 +35,251 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe
int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; }
-static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
- sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
+static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
+ return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
}
-static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
- int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
- ASSERT(sz > 0);
-#if 0
- for (int i = 0; i < sz; i++) {
- SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
- }
-#endif
+static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) {
+ return sprintf(buf, "%s/meta-ver.tmp", pWal->path);
+}
- SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1);
+static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
+ int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+ terrno = TSDB_CODE_SUCCESS;
+ ASSERT(fileIdx >= 0 && fileIdx < sz);
+
+ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr);
+ walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL);
- int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize);
- pLastFileInfo->fileSize = fileSize;
- TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
+ TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
+ wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
+ // ensure size as non-negative
+ pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize);
+
uint64_t magic = WAL_MAGIC;
+ int64_t walCkHeadSz = sizeof(SWalCkHead);
+ int64_t end = fileSize;
+ int64_t offset = 0;
+ int32_t capacity = 0;
+ int32_t readSize = 0;
+ char* buf = NULL;
+ char* found = NULL;
+ bool firstTrial = pFileInfo->fileSize < fileSize;
- char* buf = taosMemoryMalloc(readSize + 5);
- if (buf == NULL) {
- taosCloseFile(&pFile);
- terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
- return -1;
- }
-
- int64_t offset;
- offset = taosLSeekFile(pFile, -readSize, SEEK_END);
- if (readSize != taosReadFile(pFile, buf, readSize)) {
- taosMemoryFree(buf);
- taosCloseFile(&pFile);
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
-
- char* found = NULL;
+ // search for the valid last WAL entry, e.g. block by block
while (1) {
+ offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE);
+ ASSERT(offset <= end);
+ readSize = end - offset;
+ capacity = readSize + sizeof(magic);
+
+ int64_t limit = WAL_RECOV_SIZE_LIMIT;
+ if (limit < readSize) {
+ wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64
+ ", end:%" PRId64 ", file:%s",
+ pWal->cfg.vgId, limit, offset, end, fnameStr);
+ terrno = TSDB_CODE_WAL_SIZE_LIMIT;
+ goto _err;
+ }
+
+ void* ptr = taosMemoryRealloc(buf, capacity);
+ if (ptr == NULL) {
+ terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
+ goto _err;
+ }
+ buf = ptr;
+
+ int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET);
+ if (ret < 0) {
+ wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (readSize != taosReadFile(pFile, buf, readSize)) {
+ wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ readSize, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ char* candidate = NULL;
char* haystack = buf;
- char* candidate;
- while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) {
- // read and validate
- SWalCkHead* logContent = (SWalCkHead*)candidate;
- if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) {
- found = candidate;
+
+ while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) {
+ // validate head
+ int64_t len = readSize - (candidate - buf);
+ if (len < walCkHeadSz) {
+ break;
}
+ SWalCkHead* logContent = (SWalCkHead*)candidate;
+ if (walValidHeadCksum(logContent) != 0) {
+ wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s",
+ ((char*)(logContent)-buf), fnameStr);
+ haystack = candidate + 1;
+ if (firstTrial) {
+ break;
+ } else {
+ continue;
+ }
+ }
+
+ // validate body
+ int64_t size = walCkHeadSz + logContent->head.bodyLen;
+ if (len < size) {
+ int64_t extraSize = size - len;
+ if (capacity < readSize + extraSize + sizeof(magic)) {
+ capacity += extraSize;
+ void* ptr = taosMemoryRealloc(buf, capacity);
+ if (ptr == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ buf = ptr;
+ }
+ int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET);
+ if (ret < 0) {
+ wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno),
+ offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+ if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) {
+ wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s",
+ pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+ }
+ if (walValidBodyCksum(logContent) != 0) {
+ terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
+ wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s",
+ ((char*)(logContent)-buf), fnameStr);
+ haystack = candidate + 1;
+ if (firstTrial) {
+ break;
+ } else {
+ continue;
+ }
+ }
+
+ // found one
+ found = candidate;
haystack = candidate + 1;
}
+
if (found || offset == 0) break;
- offset = TMIN(0, offset - readSize + sizeof(uint64_t));
- int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET);
- ASSERT(offset == offset2);
- if (readSize != taosReadFile(pFile, buf, readSize)) {
- taosMemoryFree(buf);
- taosCloseFile(&pFile);
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
-#if 0
- if (found == buf) {
- SWalCkHead* logContent = (SWalCkHead*)found;
- if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) {
- // file has to be deleted
- taosMemoryFree(buf);
- taosCloseFile(&pFile);
- terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
- return -1;
- }
- }
-#endif
+
+ // go backwards, e.g. by at most one WAL scan buf size
+ end = offset + walCkHeadSz - 1;
+ firstTrial = false;
}
- if (found == NULL) {
- // file corrupted, no complete log
- // TODO delete and search in previous files
- /*ASSERT(0);*/
- terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
- return -1;
+ // determine end of last entry
+ SWalCkHead* lastEntry = (SWalCkHead*)found;
+ int64_t retVer = -1;
+ int64_t lastEntryBeginOffset = 0;
+ int64_t lastEntryEndOffset = 0;
+
+ if (lastEntry == NULL) {
+ terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+ } else {
+ retVer = lastEntry->head.version;
+ lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf);
+ lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen;
}
// truncate file
- SWalCkHead* lastEntry = (SWalCkHead*)found;
- int64_t retVer = lastEntry->head.version;
- int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf);
- int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen;
if (lastEntryEndOffset != fileSize) {
- wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
+ wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
fileSize);
- taosFtruncateFile(pFile, lastEntryEndOffset);
- ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset;
- pWal->totSize -= (fileSize - lastEntryEndOffset);
+ if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) {
+ wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ if (taosFsyncFile(pFile) < 0) {
+ wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
}
+ pFileInfo->fileSize = lastEntryEndOffset;
taosCloseFile(&pFile);
taosMemoryFree(buf);
-
return retVer;
+
+_err:
+ taosCloseFile(&pFile);
+ taosMemoryFree(buf);
+ return -1;
+}
+
+static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
+ int metaFileNum = taosArrayGetSize(metaLogList);
+ int actualFileNum = taosArrayGetSize(actualLogList);
+ int j = 0;
+
+ // both of the lists in asc order
+ for (int i = 0; i < actualFileNum; i++) {
+ SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i);
+ while (j < metaFileNum) {
+ SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j);
+ ASSERT(pMetaInfo != NULL);
+ if (pMetaInfo->firstVer < pLogInfo->firstVer) {
+ j++;
+ } else if (pMetaInfo->firstVer == pLogInfo->firstVer) {
+ (*pLogInfo) = *pMetaInfo;
+ j++;
+ break;
+ } else {
+ break;
+ }
+ }
+ }
+
+ taosArrayClear(metaLogList);
+
+ for (int i = 0; i < actualFileNum; i++) {
+ SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i);
+ taosArrayPush(metaLogList, pFileInfo);
+ }
+}
+
+void walAlignVersions(SWal* pWal) {
+ if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) {
+ wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId,
+ pWal->vers.firstVer, pWal->vers.snapshotVer);
+ pWal->vers.firstVer = pWal->vers.snapshotVer + 1;
+ }
+ if (pWal->vers.lastVer < pWal->vers.snapshotVer) {
+ wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+ pWal->vers.lastVer, pWal->vers.snapshotVer);
+ pWal->vers.lastVer = pWal->vers.snapshotVer;
+ }
+ if (pWal->vers.commitVer < pWal->vers.snapshotVer) {
+ wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+ pWal->vers.commitVer, pWal->vers.snapshotVer);
+ pWal->vers.commitVer = pWal->vers.snapshotVer;
+ }
+ if (pWal->vers.appliedVer < pWal->vers.snapshotVer) {
+ wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+ pWal->vers.appliedVer, pWal->vers.snapshotVer);
+ pWal->vers.appliedVer = pWal->vers.snapshotVer;
+ }
+
+ pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer);
+ pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer);
}
int walCheckAndRepairMeta(SWal* pWal) {
@@ -150,7 +288,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern;
regex_t idxRegPattern;
- bool fixed = false;
regcomp(&logRegPattern, logPattern, REG_EXTENDED);
regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
@@ -184,225 +321,237 @@ int walCheckAndRepairMeta(SWal* pWal) {
taosArraySort(actualLog, compareWalFileInfo);
- int metaFileNum = taosArrayGetSize(pWal->fileInfoSet);
- int actualFileNum = taosArrayGetSize(actualLog);
+ int metaFileNum = taosArrayGetSize(pWal->fileInfoSet);
+ int actualFileNum = taosArrayGetSize(actualLog);
+ int64_t firstVerPrev = pWal->vers.firstVer;
+ int64_t lastVerPrev = pWal->vers.lastVer;
+ int64_t totSize = 0;
+ bool updateMeta = (metaFileNum != actualFileNum);
-#if 0
- for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) {
- SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo);
+ // rebuild meta of file info
+ walRebuildFileInfoSet(pWal->fileInfoSet, actualLog);
+ taosArrayDestroy(actualLog);
+
+ int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+ ASSERT(sz == actualFileNum);
+
+ // scan and determine the lastVer
+ int32_t fileIdx = sz;
+
+ while (--fileIdx >= 0) {
char fnameStr[WAL_FILE_LEN];
+ int64_t fileSize = 0;
+ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
+
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
- int64_t fileSize = 0;
- taosStatFile(fnameStr, &fileSize, NULL);
- if (fileSize == 0) {
+ int32_t code = taosStatFile(fnameStr, &fileSize, NULL);
+ if (code < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
+ return -1;
+ }
+
+ ASSERT(pFileInfo->firstVer >= 0);
+
+ if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) {
+ totSize += pFileInfo->fileSize;
+ continue;
+ }
+ updateMeta = true;
+
+ int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx);
+ if (lastVer < 0) {
+ if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) {
+ wError("failed to scan wal last ver since %s", terrstr());
+ return -1;
+ }
+ ASSERT(pFileInfo->fileSize == 0);
+ // remove the empty wal log, and its idx
taosRemoveFile(fnameStr);
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
taosRemoveFile(fnameStr);
- taosArrayPop(pLogInfoArray);
- } else {
- break;
+ // remove its meta entry
+ taosArrayRemove(pWal->fileInfoSet, fileIdx);
+ continue;
}
+
+ // update lastVer
+ pFileInfo->lastVer = lastVer;
+ totSize += pFileInfo->fileSize;
}
- actualFileNum = taosArrayGetSize(pLogInfoArray);
-#endif
-
- {
- int32_t i = 0, j = 0;
- while (i < actualFileNum && j < metaFileNum) {
- SWalFileInfo* pActualFile = taosArrayGet(actualLog, i);
- SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j);
- if (pActualFile->firstVer < pMetaFile->firstVer) {
- char fNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- i++;
- } else if (pActualFile->firstVer > pMetaFile->firstVer) {
- taosArrayRemove(pWal->fileInfoSet, j);
- metaFileNum--;
- } else {
- i++;
- j++;
- }
- }
- if (i == actualFileNum && j == metaFileNum) {
- if (j > 0) {
- SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1);
- int64_t fsize = 0;
- char fNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pLastInfo->firstVer, fNameStr);
- taosStatFile(fNameStr, &fsize, NULL);
- if (pLastInfo->fileSize != fsize) {
- fixed = true;
- pLastInfo->fileSize = fsize;
- pLastInfo->lastVer = walScanLogGetLastVer(pWal);
- }
- }
- } else {
- fixed = true;
- while (i < actualFileNum) {
- SWalFileInfo* pActualFile = taosArrayGet(actualLog, i);
- char fNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pActualFile->firstVer, fNameStr);
- taosStatFile(fNameStr, &pActualFile->fileSize, NULL);
-
- if (pActualFile->fileSize == 0) {
- ASSERT(i == actualFileNum - 1);
- taosRemoveFile(fNameStr);
-
- walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- break;
- }
-
- if (i < actualFileNum - 1) {
- pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1;
- taosArrayPush(pWal->fileInfoSet, pActualFile);
- i++;
- } else {
- pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile);
- pActualFile->lastVer = walScanLogGetLastVer(pWal);
- if (pActualFile->lastVer == -1) {
- taosRemoveFile(fNameStr);
-
- walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- taosArrayPop(pWal->fileInfoSet);
- }
- break;
- }
- }
- }
- }
-
-#if 0
- if (metaFileNum > actualFileNum) {
- taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum);
- } else if (metaFileNum < actualFileNum) {
- for (int i = metaFileNum; i < actualFileNum; i++) {
- SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i);
- taosArrayPush(pWal->fileInfoSet, pFileInfo);
- }
- }
-#endif
-
- taosArrayDestroy(actualLog);
-
+ // reset vers info and so on
actualFileNum = taosArrayGetSize(pWal->fileInfoSet);
pWal->writeCur = actualFileNum - 1;
-
+ pWal->totSize = totSize;
+ pWal->vers.lastVer = -1;
if (actualFileNum > 0) {
- int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer;
- if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) {
- fixed = true;
- pWal->vers.lastVer = fLastVer;
- }
- int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
- if (fFirstVer != pWal->vers.firstVer) {
- fixed = true;
- pWal->vers.firstVer = fFirstVer;
- }
+ pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
+ pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer;
+ }
+ (void)walAlignVersions(pWal);
+
+ // update meta file
+ if (updateMeta) {
+ (void)walSaveMeta(pWal);
+ }
+ return 0;
+}
+
+int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
+ if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
}
- if (fixed) {
- walSaveMeta(pWal);
+ if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ if (walValidHeadCksum(pCkHead) != 0) {
+ terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
+ return -1;
}
return 0;
}
-int walCheckAndRepairIdx(SWal* pWal) {
+int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
- for (int32_t i = 0; i < sz; i++) {
- SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
+ ASSERT(fileIdx >= 0 && fileIdx < sz);
+ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
+ char fnameStr[WAL_FILE_LEN];
+ walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
+ char fLogNameStr[WAL_FILE_LEN];
+ walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
+ int64_t fileSize = 0;
- char fnameStr[WAL_FILE_LEN];
- walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
- int64_t fsize;
- TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
- if (pIdxFile == NULL) {
- ASSERT(0);
+ if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) {
+ wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer);
+ if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
+ return 0;
+ }
+
+ // start to repair
+ int64_t offset = fileSize - fileSize % sizeof(SWalIdxEntry);
+ TdFilePtr pLogFile = NULL;
+ TdFilePtr pIdxFile = NULL;
+ SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)};
+ SWalCkHead ckHead;
+ memset(&ckHead, 0, sizeof(ckHead));
+ ckHead.head.version = idxEntry.ver;
+
+ pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
+ if (pIdxFile == NULL) {
+ wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
+ if (pLogFile == NULL) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
+ goto _err;
+ }
+
+ // determine the last valid entry end, i.e. offset
+ while ((offset -= sizeof(SWalIdxEntry)) >= 0) {
+ if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
+ wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr());
- return -1;
+ goto _err;
}
- taosFStatFile(pIdxFile, &fsize, NULL);
- if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
- taosCloseFile(&pIdxFile);
+ if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
+ wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ offset, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (idxEntry.ver > pFileInfo->lastVer) {
continue;
}
- int32_t left = fsize % sizeof(SWalIdxEntry);
- int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END);
- if (left != 0) {
- taosFtruncateFile(pIdxFile, offset);
- wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr,
- offset, fsize);
- }
- offset -= sizeof(SWalIdxEntry);
-
- SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer};
- while (1) {
- if (offset < 0) {
- taosLSeekFile(pIdxFile, 0, SEEK_SET);
- taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
- break;
- }
- if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d cannot seek offset %ld when repair idx since %s", pWal->cfg.vgId, offset, terrstr());
- }
- int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
- if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
- if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) {
- taosFtruncateFile(pIdxFile, offset);
- wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld",
- pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset);
- offset -= sizeof(SWalIdxEntry);
- } else {
- break;
- }
+ if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
+ wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "",
+ pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver);
+ continue;
}
- if (idxEntry.ver < pFileInfo->lastVer) {
- char fLogNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
- TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
- if (pLogFile == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
- return -1;
- }
- while (idxEntry.ver < pFileInfo->lastVer) {
- if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset,
- terrstr());
- return -1;
- }
- SWalCkHead ckHead;
- taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead));
- if (idxEntry.ver != ckHead.head.version) {
- // todo truncate this idx also
- taosCloseFile(&pLogFile);
- wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId,
- idxEntry.offset, idxEntry.ver, ckHead.head.version);
- return -1;
- }
- idxEntry.ver = ckHead.head.version + 1;
- idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen;
- wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset);
- taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
- }
- taosCloseFile(&pLogFile);
+ if (idxEntry.ver == ckHead.head.version) {
+ break;
+ }
+ }
+ offset += sizeof(SWalIdxEntry);
+
+ // ftruncate idx file
+ if (offset < fileSize) {
+ if (taosFtruncateFile(pIdxFile, offset) < 0) {
+ wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
+ strerror(errno), offset, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ }
+
+ // rebuild idx file
+ if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) {
+ wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ offset, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ while (idxEntry.ver < pFileInfo->lastVer) {
+ ASSERT(idxEntry.ver == ckHead.head.version);
+
+ idxEntry.ver += 1;
+ idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
+
+ if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
+ wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
+ idxEntry.offset, fLogNameStr);
+ goto _err;
+ }
+ wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset);
+ if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) {
+ wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
+ goto _err;
+ }
+ }
+
+ if (taosFsyncFile(pIdxFile) < 0) {
+ wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
+ goto _err;
+ }
+
+ (void)taosCloseFile(&pLogFile);
+ (void)taosCloseFile(&pIdxFile);
+ return 0;
+
+_err:
+ (void)taosCloseFile(&pLogFile);
+ (void)taosCloseFile(&pIdxFile);
+ return -1;
+}
+
+int walCheckAndRepairIdx(SWal* pWal) {
+ int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+ int32_t fileIdx = sz;
+ while (--fileIdx >= 0) {
+ if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) {
+ wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx);
+ return -1;
}
- taosCloseFile(&pIdxFile);
}
return 0;
}
@@ -495,14 +644,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0);
cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField;
pRoot = cJSON_Parse(bytes);
+ if (!pRoot) goto _err;
pMeta = cJSON_GetObjectItem(pRoot, "meta");
+ if (!pMeta) goto _err;
pField = cJSON_GetObjectItem(pMeta, "firstVer");
+ if (!pField) goto _err;
pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pMeta, "snapshotVer");
+ if (!pField) goto _err;
pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pMeta, "commitVer");
+ if (!pField) goto _err;
pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pMeta, "lastVer");
+ if (!pField) goto _err;
pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField));
pFiles = cJSON_GetObjectItem(pRoot, "files");
@@ -512,17 +667,23 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
taosArrayEnsureCap(pArray, sz);
SWalFileInfo* pData = pArray->pData;
for (int i = 0; i < sz; i++) {
- cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
+ cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
+ if (!pInfoJson) goto _err;
SWalFileInfo* pInfo = &pData[i];
pField = cJSON_GetObjectItem(pInfoJson, "firstVer");
+ if (!pField) goto _err;
pInfo->firstVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "lastVer");
+ if (!pField) goto _err;
pInfo->lastVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "createTs");
+ if (!pField) goto _err;
pInfo->createTs = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "closeTs");
+ if (!pField) goto _err;
pInfo->closeTs = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "fileSize");
+ if (!pField) goto _err;
pInfo->fileSize = atoll(cJSON_GetStringValue(pField));
}
taosArraySetSize(pArray, sz);
@@ -530,6 +691,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
pWal->writeCur = sz - 1;
cJSON_Delete(pRoot);
return 0;
+
+_err:
+ cJSON_Delete(pRoot);
+ return -1;
}
static int walFindCurMetaVer(SWal* pWal) {
@@ -565,22 +730,63 @@ static int walFindCurMetaVer(SWal* pWal) {
int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
- walBuildMetaName(pWal, metaVer + 1, fnameStr);
- TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE);
- if (pMetaFile == NULL) {
+ char tmpFnameStr[WAL_FILE_LEN];
+ int n;
+
+ // fsync the idx and log file at first to ensure validity of meta
+ if (taosFsyncFile(pWal->pIdxFile) < 0) {
+ wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
+
+ if (taosFsyncFile(pWal->pLogFile) < 0) {
+ wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ // flush to a tmpfile
+ n = walBuildTmpMetaName(pWal, tmpFnameStr);
+ ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name");
+
+ TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+ if (pMetaFile == NULL) {
+ wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
if (len != taosWriteFile(pMetaFile, serialized, len)) {
- // TODO:clean file
-
- taosCloseFile(&pMetaFile);
- taosRemoveFile(fnameStr);
- return -1;
+ wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (taosFsyncFile(pMetaFile) < 0) {
+ wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (taosCloseFile(&pMetaFile) < 0) {
+ wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ // rename it
+ n = walBuildMetaName(pWal, metaVer + 1, fnameStr);
+ ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name");
+
+ if (taosRenameFile(tmpFnameStr, fnameStr) < 0) {
+ wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
}
- taosCloseFile(&pMetaFile);
// delete old file
if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr);
@@ -588,6 +794,11 @@ int walSaveMeta(SWal* pWal) {
}
taosMemoryFree(serialized);
return 0;
+
+_err:
+ taosCloseFile(&pMetaFile);
+ taosMemoryFree(serialized);
+ return -1;
}
int walLoadMeta(SWal* pWal) {
@@ -629,6 +840,10 @@ int walLoadMeta(SWal* pWal) {
}
// load into fileInfoSet
int code = walMetaDeserialize(pWal, buf);
+ if (code < 0) {
+ wError("failed to deserialize wal meta. file:%s", fnameStr);
+ terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+ }
taosCloseFile(&pFile);
taosMemoryFree(buf);
return code;
diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c
index 7974f3e32e..29058cada1 100644
--- a/source/libs/wal/src/walMgmt.c
+++ b/source/libs/wal/src/walMgmt.c
@@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
return NULL;
}
+ if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ taosMemoryFree(pWal);
+ return NULL;
+ }
+
// set config
memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg));
@@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
tstrncpy(pWal->path, path, sizeof(pWal->path));
if (taosMkDir(pWal->path) != 0) {
wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
- taosMemoryFree(pWal);
- return NULL;
+ goto _err;
}
// init ref
pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
if (pWal->pRefHash == NULL) {
- taosMemoryFree(pWal);
- return NULL;
+ wError("failed to init hash since %s", tstrerror(terrno));
+ goto _err;
}
// open meta
@@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
if (pWal->fileInfoSet == NULL) {
wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
- taosHashCleanup(pWal->pRefHash);
- taosMemoryFree(pWal);
- return NULL;
+ goto _err;
}
// init status
@@ -131,46 +134,37 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
pWal->writeHead.head.protoVer = WAL_PROTO_VER;
pWal->writeHead.magic = WAL_MAGIC;
- if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) {
- taosArrayDestroy(pWal->fileInfoSet);
- taosHashCleanup(pWal->pRefHash);
- taosMemoryFree(pWal);
- return NULL;
- }
-
- pWal->refId = taosAddRef(tsWal.refSetId, pWal);
- if (pWal->refId < 0) {
- taosHashCleanup(pWal->pRefHash);
- taosThreadMutexDestroy(&pWal->mutex);
- taosArrayDestroy(pWal->fileInfoSet);
- taosMemoryFree(pWal);
- return NULL;
- }
-
- walLoadMeta(pWal);
+ // load meta
+ (void)walLoadMeta(pWal);
if (walCheckAndRepairMeta(pWal) < 0) {
wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId);
- taosHashCleanup(pWal->pRefHash);
- taosRemoveRef(tsWal.refSetId, pWal->refId);
- taosThreadMutexDestroy(&pWal->mutex);
- taosArrayDestroy(pWal->fileInfoSet);
- return NULL;
+ goto _err;
}
if (walCheckAndRepairIdx(pWal) < 0) {
wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId);
- taosHashCleanup(pWal->pRefHash);
- taosRemoveRef(tsWal.refSetId, pWal->refId);
- taosThreadMutexDestroy(&pWal->mutex);
- taosArrayDestroy(pWal->fileInfoSet);
- return NULL;
+ goto _err;
+ }
+
+ // add ref
+ pWal->refId = taosAddRef(tsWal.refSetId, pWal);
+ if (pWal->refId < 0) {
+ wError("failed to add ref for Wal since %s", tstrerror(terrno));
+ goto _err;
}
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,
pWal->cfg.fsyncPeriod);
-
return pWal;
+
+_err:
+ taosArrayDestroy(pWal->fileInfoSet);
+ taosHashCleanup(pWal->pRefHash);
+ taosThreadMutexDestroy(&pWal->mutex);
+ taosMemoryFree(pWal);
+ pWal = NULL;
+ return NULL;
}
int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
@@ -195,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
void walClose(SWal *pWal) {
taosThreadMutexLock(&pWal->mutex);
+ (void)walSaveMeta(pWal);
taosCloseFile(&pWal->pLogFile);
pWal->pLogFile = NULL;
taosCloseFile(&pWal->pIdxFile);
pWal->pIdxFile = NULL;
- walSaveMeta(pWal);
taosArrayDestroy(pWal->fileInfoSet);
pWal->fileInfoSet = NULL;
taosHashCleanup(pWal->pRefHash);
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index b2cd4fac11..179d809c84 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
- ASSERT(pRet != NULL);
+ if (pRet == NULL) {
+ wError("failed to find WAL log file with ver:%lld", ver);
+ terrno = TSDB_CODE_WAL_INVALID_VER;
+ return -1;
+ }
if (pReader->curFileFirstVer != pRet->firstVer) {
// error code was set inner
if (walReadChangeFile(pReader, pRet->firstVer) < 0) {
@@ -472,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
- ASSERT(0);
+ wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s",
+ pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}
@@ -505,6 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
+ wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s",
+ pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}
diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c
index 1196914dae..4b75db52b7 100644
--- a/source/libs/wal/src/walSeek.c
+++ b/source/libs/wal/src/walSeek.c
@@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->pLogFile != NULL) {
+ code = taosFsyncFile(pWal->pLogFile);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
}
}
if (pWal->pIdxFile != NULL) {
+ code = taosFsyncFile(pWal->pIdxFile);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index e7161079d9..91fa49fce0 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -45,6 +45,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
+ taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
@@ -53,6 +54,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
+ taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
@@ -219,10 +221,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
taosCloseFile(&pIdxFile);
taosCloseFile(&pLogFile);
- taosFsyncFile(pWal->pLogFile);
- taosFsyncFile(pWal->pIdxFile);
-
- walSaveMeta(pWal);
+ code = walSaveMeta(pWal);
+ if (code < 0) {
+ wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
+ taosThreadMutexUnlock(&pWal->mutex);
+ return -1;
+ }
// unlock
taosThreadMutexUnlock(&pWal->mutex);
@@ -394,7 +398,11 @@ int32_t walRollImpl(SWal *pWal) {
pWal->lastRollSeq = walGetSeq();
- walSaveMeta(pWal);
+ code = walSaveMeta(pWal);
+ if (code < 0) {
+ wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
+ goto END;
+ }
END:
return code;
@@ -550,6 +558,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
void walFsync(SWal *pWal, bool forceFsync) {
if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
+ wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
+ if (taosFsyncFile(pWal->pIdxFile) < 0) {
+ wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
+ strerror(errno));
+ }
wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
if (taosFsyncFile(pWal->pLogFile) < 0) {
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c
index f0442c6fd1..616ab7875d 100644
--- a/source/os/src/osEnv.c
+++ b/source/os/src/osEnv.c
@@ -87,11 +87,17 @@ void osUpdate() {
void osCleanup() {}
-bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; }
+bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; }
-bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; }
+bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; }
-bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; }
+bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; }
+
+bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; }
+
+bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; }
+
+bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; }
void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); }
diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c
index 7e1506c140..f3ccbb0aac 100644
--- a/source/util/src/tbloomfilter.c
+++ b/source/util/src/tbloomfilter.c
@@ -19,12 +19,12 @@
#include "taos.h"
#include "taoserror.h"
-#define UNIT_NUM_BITS 64
-#define UNIT_ADDR_NUM_BITS 6
+#define UNIT_NUM_BITS 64ULL
+#define UNIT_ADDR_NUM_BITS 6ULL
static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
- uint64_t mask = 1 << (index % UNIT_NUM_BITS);
+ uint64_t mask = 1ULL << (index % UNIT_NUM_BITS);
uint64_t old = buf[unitIndex];
buf[unitIndex] |= mask;
return buf[unitIndex] != old;
@@ -32,7 +32,7 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
- uint64_t mask = 1 << (index % UNIT_NUM_BITS);
+ uint64_t mask = 1ULL << (index % UNIT_NUM_BITS);
return buf[unitIndex] & mask;
}
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index eb13a08be4..9a117c6eb4 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -447,12 +447,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f
TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset")
// wal
-TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Wal unexpected generic error")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch")
// tfs
TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config")
diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
index 0a835ec564..c68375894c 100644
--- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
+++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
@@ -32,6 +32,10 @@ class TDTestCase:
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
+ elif "src" in selfPath:
+ projPath = selfPath[: selfPath.find("src")]
+ elif "/tools/" in selfPath:
+ projPath = selfPath[: selfPath.find("/tools/")]
else:
projPath = selfPath[: selfPath.find("tests")]
@@ -55,11 +59,10 @@ class TDTestCase:
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
tdSql.execute("reset query cache")
- tdSql.query("select count(*) from (select distinct(tbname) from db.stb)")
- tdSql.checkData(0, 0, 10)
+ tdSql.query("show db.tables")
+ tdSql.checkRows(10)
tdSql.query("select count(*) from db.stb")
- if len(tdSql.queryResult):
- tdLog.exit("query result is %d" % len(tdSql.queryResult))
+ tdSql.checkData(0, 0, 100)
def stop(self):
tdSql.close()
diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
index d4b2aae2fb..da22ef75e2 100644
--- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
+++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
@@ -20,8 +20,10 @@
"super_tables": [{
"name": "stb",
"childtable_prefix": "stb_",
+ "childtable_count": 10,
+ "insert_rows": 10,
"columns": [{"type": "INT"}],
"tags": [{"type": "INT"}]
}]
}]
-}
\ No newline at end of file
+}
diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh
new file mode 100755
index 0000000000..457c9eae20
--- /dev/null
+++ b/tests/script/coverage_test.sh
@@ -0,0 +1,291 @@
+#!/bin/bash
+
+branch=
+if [ x$1 != x ];then
+ branch=$1
+ echo "Testing branch: $branch"
+else
+ echo "Please enter branch name as a parameter"
+ exit 1
+fi
+
+today=`date +"%Y%m%d"`
+TDENGINE_DIR=/root/pxiao/TDengine
+TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDengine {
+ echo "check if TDengine need build"
+ cd $TDENGINE_DIR
+ git remote prune origin > /dev/null
+ git remote update > /dev/null
+ REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch`
+ LOCAL_COMMIT=`git rev-parse --short @`
+ echo " LOCAL: $LOCAL_COMMIT"
+ echo "REMOTE: $REMOTE_COMMIT"
+
+ # reset counter
+ lcov -d . --zerocounters
+
+
+ if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+ echo "repo up-to-date"
+ else
+ echo "repo need to pull"
+ fi
+
+ git reset --hard
+ git checkout -- .
+ git checkout $branch
+ git clean -dfx
+ git pull
+ git submodule update --init --recursive -f
+
+ [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
+ cd $TDENGINE_DIR/debug
+
+ echo "rebuild.."
+ LOCAL_COMMIT=`git rev-parse --short @`
+
+ rm -rf *
+ if [ "$branch" == "3.0" ]; then
+ echo "3.0 ============="
+ cmake -DCOVER=true -DBUILD_TEST=true ..
+ else
+ cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null
+ fi
+ make -j4
+ make install
+}
+
+function runGeneralCaseOneByOne {
+ while read -r line; do
+ if [[ $line =~ ^./test.sh* ]]; then
+ case=`echo $line | grep sim$ | awk '{print $NF}'`
+
+ if [ -n "$case" ]; then
+ date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \
+ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \
+ || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+ fi
+ fi
+ done < $1
+}
+
+function runTestNGCaseOneByOne {
+ while read -r line; do
+ if [[ $line =~ ^taostest* ]]; then
+ case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2`
+ yaml=`echo $line | cut -d' ' -f 2`
+
+ if [ -n "$case" ]; then
+ date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \
+ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \
+ || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+ fi
+ fi
+ done < $1
+}
+
+function runTest {
+ echo "run Test"
+
+ if [ "$branch" == "3.0" ]; then
+ echo "start run unit test case ................"
+ echo " $TDENGINE_DIR/debug "
+ cd $TDENGINE_DIR/debug
+ ctest -j12
+ echo "3.0 unit test done"
+ fi
+
+ cd $TDENGINE_DIR/tests/script
+
+ [ -d ../../sim ] && rm -rf ../../sim
+ [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT
+
+ runGeneralCaseOneByOne jenkins/basic.txt
+
+ sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT
+
+ totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l`
+ if [ "$totalSuccess" -gt "0" ]; then
+ sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+
+ totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l`
+ if [ "$totalFailed" -ne "0" ]; then
+ sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+ sed "3G" $TDENGINE_COVERAGE_REPORT
+
+ stopTaosd
+ echo "run TestNG cases"
+ rm -rf /var/lib/taos/*
+ rm -rf /var/log/taos/*
+ nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
+ sleep 10
+ cd $TDENGINE_DIR/../TestNG/cases
+ runTestNGCaseOneByOne ../scripts/cases.txt
+ echo "TestNG cases done"
+
+ cd $TDENGINE_DIR/tests
+ rm -rf ../sim
+ /root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT
+
+
+ sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT
+ totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
+ if [ "$totalPySuccess" -gt "0" ]; then
+ sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+
+ totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
+ if [ -z $totalPyFailed ]; then
+ sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT
+ else
+ sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+
+ echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT
+ # Test Connector
+ stopTaosd
+ nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
+ sleep 10
+
+ cd $TDENGINE_DIR/src/connector/jdbc
+ mvn clean package > /dev/null 2>&1
+ mvn test > jdbc-out.log 2>&1
+ tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+ # Test C Demo
+ stopTaosd
+ $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
+ sleep 10
+ yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+ # Test waltest
+ dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
+ walDir=`find $dataDir -name "wal"|head -n1`
+ echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
+ echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
+ if [ -n "$walDir" ]; then
+ yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+ fi
+
+ # run Unit Test
+ echo "Run Unit Test: utilTest, queryTest and cliTest"
+ #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!"
+ #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!"
+ #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!"
+
+ stopTaosd
+
+ cd $TDENGINE_DIR/tests/script
+ find . -name '*.sql' | xargs rm -f
+
+ cd $TDENGINE_DIR/tests/pytest
+ find . -name '*.sql' | xargs rm -f
+}
+
+function lcovFunc {
+ echo "collect data by lcov"
+ cd $TDENGINE_DIR
+
+ # collect data
+ lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
+
+ # remove exclude paths
+ if [ "$branch" == "3.0" ]; then
+ lcov --remove coverage.info \
+ '*/contrib/*' '*/tests/*' '*/test/*'\
+ '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\
+ '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\
+ '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\
+ '*/tthread.c' '*/tversion.c'\
+ --rc lcov_branch_coverage=1 -o coverage.info
+ else
+ lcov --remove coverage.info \
+ '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\
+ --rc lcov_branch_coverage=1 -o coverage.info
+ fi
+
+
+ # generate result
+ echo "generate result"
+ lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
+
+ # push result to coveralls.io
+ echo "push result to coveralls.io"
+ /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT
+
+ #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT
+ #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT
+ cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT
+
+}
+
+function sendReport {
+ echo "send report"
+ receiver="develop@taosdata.com"
+ mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+ cd $TDENGINE_DIR
+
+ sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT
+ BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT`
+ echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+ (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \
+ /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+ echo "Stop taosd start"
+ systemctl stop taosd
+ PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+ while [ -n "$PID" ]
+ do
+ pkill -TERM -x taosd
+ sleep 1
+ PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+ done
+ echo "Stop tasod end"
+}
+
+function runTestRandomFail {
+ exec_random_fail_sh=$1
+ default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh
+ [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1
+
+ dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py
+ default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py
+ [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1
+
+ runTest NoRandomFail
+}
+
+WORK_DIR=/root/pxiao
+
+date >> $WORK_DIR/cron.log
+echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log
+
+stopTaosd
+buildTDengine
+
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh
+
+runTest
+
+lcovFunc
+#sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 89e26e785b..d8f4a36261 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -155,7 +155,7 @@
./test.sh -f tsim/parser/select_with_tags.sim
./test.sh -f tsim/parser/selectResNum.sim
./test.sh -f tsim/parser/set_tag_vals.sim
-./test.sh -f tsim/parser/single_row_in_tb.sim
+# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim
./test.sh -f tsim/parser/sliding.sim
./test.sh -f tsim/parser/slimit_alter_tags.sim
./test.sh -f tsim/parser/slimit.sim
@@ -194,7 +194,7 @@
./test.sh -f tsim/mnode/basic1.sim
./test.sh -f tsim/mnode/basic2.sim
./test.sh -f tsim/mnode/basic3.sim
-# TD-17919 ./test.sh -f tsim/mnode/basic4.sim
+./test.sh -f tsim/mnode/basic4.sim
./test.sh -f tsim/mnode/basic5.sim
# ---- show ----
diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim
index 59156080c8..31fc276298 100644
--- a/tests/script/tsim/mnode/basic1.sim
+++ b/tests/script/tsim/mnode/basic1.sim
@@ -42,6 +42,7 @@ sql_error drop mnode on dnode 1
print =============== create mnode 2
sql create mnode on dnode 2
+print =============== create mnode 2 finished
$x = 0
step2:
$x = $x + 1
@@ -69,9 +70,10 @@ if $data(2)[2] != follower then
goto step2
endi
-sleep 2000
print ============ drop mnode 2
sql drop mnode on dnode 2
+
+print ============ drop mnode 2 finished
sql select * from information_schema.ins_mnodes
if $rows != 1 then
return -1
@@ -109,6 +111,8 @@ sleep 2000
print =============== create mnodes
sql create mnode on dnode 2
+
+print =============== create mnode 2 finished
sql select * from information_schema.ins_mnodes
if $rows != 2 then
return -1
diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim
index a2b8aa2f96..6739a87b42 100644
--- a/tests/script/tsim/mnode/basic4.sim
+++ b/tests/script/tsim/mnode/basic4.sim
@@ -45,49 +45,12 @@ endi
if $data(2)[4] != ready then
goto step2
endi
-
-system sh/exec.sh -n dnode3 -s stop
-sql_error create mnode on dnode 3
-
-print =============== step3: select * from information_schema.ins_mnodes
-
-$x = 0
-step3:
- $x = $x + 1
- sleep 1000
- if $x == 10 then
- return -1
- endi
-sql select * from information_schema.ins_mnodes -x step3
-print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4]
-print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4]
-
-$leaderNum = 0
-if $data(1)[2] == leader then
- $leaderNum = 1
-endi
-if $data(2)[2] == leader then
- $leaderNum = 1
-endi
-if $leaderNum == 0 then
- goto step3
-endi
-if $data(3)[2] != offline then
- goto step3
-endi
-if $data(1)[3] != ready then
- goto step3
-endi
-if $data(2)[3] != ready then
- goto step3
-endi
-if $data(3)[3] != creating then
- goto step3
+if $data(3)[4] != ready then
+ goto step2
endi
-print =============== step4: start dnode3
-system sh/exec.sh -n dnode3 -s start
+print =============== step4: create mnode 3
+sql create mnode on dnode 3
$x = 0
step4:
@@ -159,12 +122,11 @@ endi
if $data(2)[3] != ready then
goto step5
endi
-if $data(3)[3] != dropping then
- goto step5
-endi
print =============== step6: start dnode3
system sh/exec.sh -n dnode3 -s start
+sql drop mnode on dnode 1 -x step60
+step60:
$x = 0
step6:
diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim
index 387f38a717..16e8fa3dfa 100644
--- a/tests/script/tsim/mnode/basic5.sim
+++ b/tests/script/tsim/mnode/basic5.sim
@@ -232,12 +232,11 @@ endi
if $leaderNum != 1 then
goto step81
endi
-if $data(1)[3] != dropping then
- goto step81
-endi
print =============== step9: start mnode1 and wait it dropped
system sh/exec.sh -n dnode1 -s start
+sql drop mnode on dnode 1 -x step90
+step90:
$x = 0
step91:
diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim
index 6fa7e39335..252ef10b1a 100644
--- a/tests/script/tsim/valgrind/checkError3.sim
+++ b/tests/script/tsim/valgrind/checkError3.sim
@@ -62,7 +62,16 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start -v
print =============== stepa: query data
-sql select * from c1
+
+$x = 0
+steps:
+ $x = $x + 1
+ sleep 500
+ if $x == 50 then
+ return -1
+ endi
+sql select * from c1 -x steps
+
sql select * from stb
sql select * from stb_1
sql select ts, c1, c2, c3 from c1
diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim
index 11a387ed4d..89b6cadef8 100644
--- a/tests/script/tsim/valgrind/checkError6.sim
+++ b/tests/script/tsim/valgrind/checkError6.sim
@@ -158,9 +158,15 @@ print =============== restart
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start -v
-sleep 1000
+$x = 0
+steps:
+ $x = $x + 1
+ sleep 500
+ if $x == 50 then
+ return -1
+ endi
+sql select avg(tbcol) as c from stb -x steps
-sql select avg(tbcol) as c from stb
sql select avg(tbcol) as c from stb where ts <= 1601481840000
sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000
sql select avg(tbcol) as c from stb interval(1m)
diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py
new file mode 100644
index 0000000000..d3a55ee0a6
--- /dev/null
+++ b/tests/system-test/1-insert/alter_database.py
@@ -0,0 +1,58 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(),logSql)
+ self.buffer_boundary = [3,4097,8193,12289,16384]
+ self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96]
+ # pages_boundary >= 64
+ self.pages_boundary = [64,128,512]
+ self.pages_error = [self.pages_boundary[0]-1]
+ def alter_buffer(self):
+ tdSql.execute('create database db')
+ for buffer in self.buffer_boundary:
+ tdSql.execute(f'alter database db buffer {buffer}')
+ tdSql.query('select * from information_schema.ins_databases where name = "db"')
+ tdSql.checkEqual(tdSql.queryResult[0][8],buffer)
+ tdSql.execute('drop database db')
+ tdSql.execute('create database db vgroups 10')
+ for buffer in self.buffer_error:
+ tdSql.error(f'alter database db buffer {buffer}')
+ tdSql.execute('drop database db')
+
+ def alter_pages(self):
+ tdSql.execute('create database db')
+ for pages in self.pages_boundary:
+ tdSql.execute(f'alter database db pages {pages}')
+ tdSql.query('select * from information_schema.ins_databases where name = "db"')
+ tdSql.checkEqual(tdSql.queryResult[0][10],pages)
+ tdSql.execute('drop database db')
+ tdSql.execute('create database db')
+ tdSql.query('select * from information_schema.ins_databases where name = "db"')
+ self.pages_error.append(tdSql.queryResult[0][10])
+ for pages in self.pages_error:
+ tdSql.error(f'alter database db pages {pages}')
+ tdSql.execute('drop database db')
+
+ def run(self):
+ tdSql.error('create database db1 vgroups 10 buffer 12289')
+ self.alter_buffer()
+ self.alter_pages()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py
new file mode 100644
index 0000000000..d18cb0260e
--- /dev/null
+++ b/tests/system-test/7-tmq/create_wrong_topic.py
@@ -0,0 +1,77 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+from util.common import *
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.sqlset import *
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(),logSql)
+ self.setsql = TDSetSql()
+ self.rowNum = 10
+ self.ts = 1537146000000
+ self.binary_str = 'taosdata'
+ self.nchar_str = '涛思数据'
+ self.column_dict = {
+ 'ts' : 'timestamp',
+ 'col1': 'tinyint',
+ 'col2': 'smallint',
+ 'col3': 'int',
+ 'col4': 'bigint',
+ 'col5': 'tinyint unsigned',
+ 'col6': 'smallint unsigned',
+ 'col7': 'int unsigned',
+ 'col8': 'bigint unsigned',
+ 'col9': 'float',
+ 'col10': 'double',
+ }
+ self.error_topic = ['avg','count','spread','stddev','sum','hyperloglog']
+ def insert_data(self,column_dict,tbname,row_num):
+ insert_sql = self.setsql.set_insertsql(column_dict,tbname)
+ for i in range(row_num):
+ insert_list = []
+ self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts)
+ def wrong_topic(self):
+ tdSql.prepare()
+ tdSql.execute('use db')
+ stbname = f'db.{tdCom.getLongName(5, "letters")}'
+ tag_dict = {
+ 't0':'int'
+ }
+ tag_values = [
+ f'1'
+ ]
+ tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict))
+ tdSql.execute(f"create table {stbname}_tb1 using {stbname} tags({tag_values[0]})")
+ self.insert_data(self.column_dict,f'{stbname}_tb1',self.rowNum)
+ for column in self.column_dict.keys():
+ for func in self.error_topic:
+ if func.lower() != 'count' and column.lower() != 'ts':
+ tdSql.error(f'create topic tpn as select {func}({column}) from {stbname}')
+ elif func.lower() == 'count' :
+ tdSql.error(f'create topic tpn as select {func}(*) from {stbname}')
+ for column in self.column_dict.keys():
+ if column.lower() != 'ts':
+ tdSql.error(f'create topic tpn as select apercentile({column},50) from {stbname}')
+ tdSql.error(f'create topic tpn as select leastquares({column},1,1) from {stbname}_tb1')
+ tdSql.error(f'create topic tpn as select HISTOGRAM({column},user_input,[1,3,5,7],0) from {stbname}')
+ tdSql.error(f'create topic tpn as select percentile({column},1) from {stbname}_tb1')
+ pass
+ def run(self):
+ self.wrong_topic()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py
index a613f11267..de95fef148 100644
--- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py
+++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py
@@ -52,7 +52,7 @@ class TDTestCase:
paraDict['rowsPerTbl'] = self.rowsPerTbl
tmqCom.initConsumerTable()
- tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
+ tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1)
tdLog.info("create stb")
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
# tdLog.info("create ctb")
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 4c823faa8a..39adfc9d53 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -2,7 +2,7 @@
set -e
set -x
-#python3 ./test.py -f 0-others/taosShell.py
+python3 ./test.py -f 0-others/taosShell.py
python3 ./test.py -f 0-others/taosShellError.py
python3 ./test.py -f 0-others/taosShellNetChk.py
python3 ./test.py -f 0-others/telemetry.py
@@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/sysinfo.py
python3 ./test.py -f 0-others/user_control.py
python3 ./test.py -f 0-others/fsync.py
python3 ./test.py -f 0-others/compatibility.py
-
+python3 ./test.py -f 1-insert/alter_database.py
python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py
python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py
@@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
-# unstable python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3
@@ -248,8 +248,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 -
python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5
# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py
-# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
-# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
+python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3
@@ -279,7 +279,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_
python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1
# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1
-
+python3 ./test.py -f 7-tmq/create_wrong_topic.py
python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
python3 ./test.py -f 7-tmq/basic5.py
python3 ./test.py -f 7-tmq/subscribeDb.py
diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c
index eedb6d7295..4d40de66bd 100644
--- a/tools/shell/src/shellArguments.c
+++ b/tools/shell/src/shellArguments.c
@@ -22,7 +22,7 @@
#define TAOS_CONSOLE_PROMPT_HEADER "taos> "
#define TAOS_CONSOLE_PROMPT_CONTINUE " -> "
-#define SHELL_HOST "The auth string to use when connecting to the server."
+#define SHELL_HOST "TDengine server FQDN to connect. The default host is localhost."
#define SHELL_PORT "The TCP/IP port number to use for the connection."
#define SHELL_USER "The user name to use when connecting to the server."
#define SHELL_PASSWORD "The password to use when connecting to the server."
diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c
index 8aafd747af..46eaf6cff1 100644
--- a/tools/shell/src/shellAuto.c
+++ b/tools/shell/src/shellAuto.c
@@ -325,24 +325,23 @@ int cntDel = 0; // delete byte count after next press tab
// show auto tab introduction
void printfIntroduction() {
- printf(" **************************** How To Use TAB Key ********************************\n");
- printf(" * TDengine Command Line supports pressing TAB key to complete word, *\n");
- printf(" * including database name, table name, function name and keywords. *\n");
- printf(" * Press TAB key anywhere, you'll get surprise. *\n");
- printf(" * KEYBOARD SHORTCUT: *\n");
- printf(" * [ TAB ] ...... Complete the word or show help if no input *\n");
- printf(" * [ Ctrl + A ] ...... move cursor to [A]head of line *\n");
- printf(" * [ Ctrl + E ] ...... move cursor to [E]nd of line *\n");
- printf(" * [ Ctrl + W ] ...... move cursor to line of middle *\n");
- printf(" * [ Ctrl + L ] ...... clean screen *\n");
- printf(" * [ Ctrl + K ] ...... clean after cursor *\n");
- printf(" * [ Ctrl + U ] ...... clean before cursor *\n");
- printf(" * *\n");
+ printf(" ****************************** Tab Completion **********************************\n");
+ printf(" * The TDengine CLI supports tab completion for a variety of items, *\n");
+ printf(" * including database names, table names, function names and keywords. *\n");
+ printf(" * The full list of shortcut keys is as follows: *\n");
+ printf(" * [ TAB ] ...... complete the current word *\n");
+ printf(" * ...... if used on a blank line, display all valid commands *\n");
+ printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n");
+ printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n");
+ printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n");
+ printf(" * [ Ctrl + L ] ...... clear the entire screen *\n");
+ printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n");
+ printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n");
printf(" **********************************************************************************\n\n");
}
void showHelp() {
- printf("\nThe following are supported commands for TDengine Command Line:");
+ printf("\nThe TDengine CLI supports the following commands:");
printf(
"\n\
----- A ----- \n\
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 28375d8a35..37d00c0a7c 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) {
printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6);
// call auto tab
- callbackAutoTab(command, pSql, false);
+ callbackAutoTab(command, NULL, false);
}
printf("\r\n");
diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c
index 2f1ee12d54..b310281119 100644
--- a/tools/shell/src/shellTire.c
+++ b/tools/shell/src/shellTire.c
@@ -243,8 +243,8 @@ void enumAllWords(STireNode** nodes, char* prefix, SMatch* match) {
continue;
} else {
// combine word string
- memset(word, 0, sizeof(word));
- strcpy(word, prefix);
+ memset(word, 0, tListLen(word));
+ strncpy(word, prefix, len);
word[len] = FIRST_ASCII + i; // append current char
// chain middle node
@@ -315,8 +315,7 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) {
}
}
- // return
- return;
+ taosMemoryFree(root);
}
SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) {
diff --git a/tools/taosws-rs b/tools/taosws-rs
new file mode 160000
index 0000000000..7a94ffab45
--- /dev/null
+++ b/tools/taosws-rs
@@ -0,0 +1 @@
+Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6
diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c
index 56ba622a9c..b6d8d75ba0 100644
--- a/utils/test/c/sml_test.c
+++ b/utils/test/c/sml_test.c
@@ -119,7 +119,7 @@ int smlProcess_json1_Test() {
" \"dc\": \"lga\""
" }"
" }"
- "]"};
+ "]",};
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL,
TSDB_SML_TIMESTAMP_NANO_SECONDS);
printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
@@ -159,7 +159,7 @@ int smlProcess_json2_Test() {
" },"
" \"id\": \"d1001\""
" }"
- "}"};
+ "}",};
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL,
TSDB_SML_TIMESTAMP_NANO_SECONDS);
printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
@@ -227,7 +227,7 @@ int smlProcess_json3_Test() {
" },"
" \"id\": \"d1001\""
" }"
- "}"};
+ "}",};
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL,
TSDB_SML_TIMESTAMP_NANO_SECONDS);
printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
@@ -286,7 +286,7 @@ int smlProcess_json4_Test() {
" \"t9\": false,"
" \"id\": \"d1001\""
" }"
- "}"};
+ "}",};
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL,
TSDB_SML_TIMESTAMP_NANO_SECONDS);
printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c
index a043aa7a6d..d36fe0855a 100644
--- a/utils/test/c/tmqSim.c
+++ b/utils/test/c/tmqSim.c
@@ -155,7 +155,7 @@ static void printHelp() {
printf("%s%s\n", indent, "-l");
- printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes);
+ printf("%s%s%s%d\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes);
printf("%s%s\n", indent, "-p");
printf("%s%s%s\n", indent, indent, "producer thread number, default is 0");
printf("%s%s\n", indent, "-b");
@@ -238,7 +238,7 @@ void saveConfigToLogFile() {
taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]);
}
taosFprintfFile(g_fp, "\n");
- taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt);
+ taosFprintfFile(g_fp, " expect rows: %" PRIx64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt);
}
char tmpString[128];
@@ -263,11 +263,11 @@ void parseArgument(int32_t argc, char* argv[]) {
printHelp();
exit(0);
} else if (strcmp(argv[i], "-d") == 0) {
- strcpy(g_stConfInfo.dbName, argv[++i]);
+ tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName));
} else if (strcmp(argv[i], "-w") == 0) {
- strcpy(g_stConfInfo.cdbName, argv[++i]);
+ tstrncpy(g_stConfInfo.cdbName, argv[++i], sizeof(g_stConfInfo.cdbName));
} else if (strcmp(argv[i], "-c") == 0) {
- strcpy(configDir, argv[++i]);
+ tstrncpy(configDir, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-g") == 0) {
g_stConfInfo.showMsgFlag = atol(argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) {
@@ -279,9 +279,9 @@ void parseArgument(int32_t argc, char* argv[]) {
} else if (strcmp(argv[i], "-e") == 0) {
g_stConfInfo.useSnapshot = atol(argv[++i]);
} else if (strcmp(argv[i], "-t") == 0) {
- char tmpBuf[56];
- strcpy(tmpBuf, argv[++i]);
- sprintf(g_stConfInfo.topic, "`%s`", tmpBuf);
+ char tmpBuf[56] = {0};
+ tstrncpy(tmpBuf, argv[++i], sizeof(tmpBuf));
+ sprintf(g_stConfInfo.topic, "`%s`", tmpBuf);
} else if (strcmp(argv[i], "-x") == 0) {
g_stConfInfo.numOfThread = atol(argv[++i]);
} else if (strcmp(argv[i], "-l") == 0) {
@@ -294,6 +294,10 @@ void parseArgument(int32_t argc, char* argv[]) {
g_stConfInfo.producerRate = atol(argv[++i]);
} else if (strcmp(argv[i], "-n") == 0) {
g_stConfInfo.payloadLen = atol(argv[++i]);
+ if(g_stConfInfo.payloadLen <= 0 || g_stConfInfo.payloadLen > 1024 * 1024 * 1024){
+ pError("%s calloc size is too large: %s %s", GREEN, argv[++i], NC);
+ exit(-1);
+ }
} else {
pError("%s unknow para: %s %s", GREEN, argv[++i], NC);
exit(-1);
@@ -354,8 +358,8 @@ void ltrim(char* str) {
int queryDB(TAOS* taos, char* command) {
int retryCnt = 10;
- int code;
- TAOS_RES* pRes;
+ int code = 0;
+ TAOS_RES* pRes = NULL;
while (retryCnt--) {
pRes = taos_query(taos, command);
@@ -363,10 +367,11 @@ int queryDB(TAOS* taos, char* command) {
if (code != 0) {
taosSsleep(1);
taos_free_result(pRes);
+ pRes = NULL;
continue;
}
taos_free_result(pRes);
- return 0;
+ return 0;
}
pError("failed to reason:%s, sql: %s", tstrerror(code), command);
@@ -418,7 +423,7 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) {
char sqlStr[1100] = {0};
if (strlen(buf) > 1024) {
- taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf));
+ taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", (int)strlen(buf));
taosCloseFile(&g_fp);
return -1;
}
@@ -592,7 +597,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
int32_t vgroupId = tmq_get_vgroup_id(msg);
const char* dbName = tmq_get_db_name(msg);
- taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex);
+ taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex);
taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table",
tmq_get_topic_name(msg), vgroupId);
@@ -644,7 +649,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
int32_t vgroupId = tmq_get_vgroup_id(msg);
const char* dbName = tmq_get_db_name(msg);
- taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex);
+ taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex);
taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table",
tmq_get_topic_name(msg), vgroupId);
@@ -960,7 +965,7 @@ void parseConsumeInfo() {
ltrim(pstr);
char* ret = strchr(pstr, ch);
memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr);
- strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1);
+ tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1, sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey]));
// printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey],
// g_stConfInfo.value[g_stConfInfo.numOfKey]);
g_stConfInfo.stThreads[i].numOfKey++;
@@ -1268,25 +1273,26 @@ void* ombProduceThreadFunc(void* param) {
for (int i = 0; i < batchPerTblTimes; ++i) {
uint32_t msgsOfSql = g_stConfInfo.batchSize;
if ((i == batchPerTblTimes - 1) && (0 != remainder)) {
- msgsOfSql = remainder;
+ msgsOfSql = remainder;
}
int len = 0;
len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName);
for (int j = 0; j < msgsOfSql; j++) {
- int64_t timeStamp = taosGetTimestampNs();
- len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload);
+ int64_t timeStamp = taosGetTimestampNs();
+ len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload);
sendMsgs++;
pInfo->totalProduceMsgs++;
}
- totalMsgLen += len;
+ totalMsgLen += len;
pInfo->totalMsgsLen += len;
- int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE);
+ int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE);
if (affectedRows < 0) {
taos_close(pInfo->taos);
- pInfo->taos = NULL;
- return NULL;
+ pInfo->taos = NULL;
+ taosMemoryFree(sqlBuf);
+ return NULL;
}
affectedRowsTotal += affectedRows;
@@ -1322,6 +1328,7 @@ void* ombProduceThreadFunc(void* param) {
printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal);
taos_close(pInfo->taos);
pInfo->taos = NULL;
+ taosMemoryFree(sqlBuf);
return NULL;
}
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 3a598ba98b..6540fdac4c 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -663,7 +663,7 @@ void initLogFile() {
int main(int argc, char* argv[]) {
for (int32_t i = 1; i < argc; i++) {
if(strcmp(argv[i], "-c") == 0){
- strcpy(g_conf.dir, argv[++i]);
+ tstrncpy(g_conf.dir, argv[++i], sizeof(g_conf.dir));
}else if(strcmp(argv[i], "-s") == 0){
g_conf.snapShot = true;
}else if(strcmp(argv[i], "-d") == 0){