diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 7bde332c8c..f0a1ad83ad 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG d58230c
+ GIT_TAG 2849aa4
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md
index d95086d4c4..6295d8553d 100644
--- a/docs/en/14-reference/05-taosbenchmark.md
+++ b/docs/en/14-reference/05-taosbenchmark.md
@@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js
- **name**: specify the name of the database.
-- **drop**: indicate whether to delete the database before inserting. The default is true.
+- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop.
#### Stream processing related configuration parameters
diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md
index 4c33b5232a..9346c3f763 100644
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@@ -340,7 +340,7 @@ LTRIM(expr)
#### RTRIM
```sql
-LTRIM(expr)
+RTRIM(expr)
```
**功能说明**：返回清除右边空格后的字符串。
diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md
index 7da8e9f331..1ab7f4f016 100644
--- a/docs/zh/12-taos-sql/29-changes.md
+++ b/docs/zh/12-taos-sql/29-changes.md
@@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明"
| 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
| 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
| 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
+
+
+## SCHEMALESS 变更
+
+| # | **元素** | **差异性
** | **说明** |
+| - | :------- | :-------- | :------- |
+| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。
diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md
index 9f4f728f78..e067c646b2 100644
--- a/docs/zh/14-reference/05-taosbenchmark.md
+++ b/docs/zh/14-reference/05-taosbenchmark.md
@@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
- **name** : 数据库名。
-- **drop** : 插入前是否删除数据库，默认为 true。
+- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。
#### 流式计算相关配置参数
diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md
index 07d5b0692b..b89f625376 100644
--- a/docs/zh/14-reference/13-schemaless/13-schemaless.md
+++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md
@@ -115,7 +115,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000
## 数据模式映射规则
-本节将说明行协议的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为
+本节将说明 InfluxDB 行协议(Line Protocol)的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为
超级表名称。tag_set 中的 标签名称为 数据模式中的标签名，field_set 中的名称为列名称。以如下数据为例，说明映射规则：
```json
diff --git a/examples/c/demoapi.c b/examples/c/demoapi.c
index c3a6d26776..18f1b5a059 100644
--- a/examples/c/demoapi.c
+++ b/examples/c/demoapi.c
@@ -1,5 +1,17 @@
-// C api call sequence demo
-// to compile: gcc -o apidemo apidemo.c -ltaos
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
#include
#include
@@ -13,23 +25,23 @@
#include "taos.h"
#define debugPrint(fmt, ...) \
- do { if (g_args.debug_print || g_args.verbose_print) \
- fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0)
+ do { if (g_args.debug_print || g_args.verbose_print) {\
+ fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0)
#define warnPrint(fmt, ...) \
do { fprintf(stderr, "\033[33m"); \
fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \
- fprintf(stderr, "\033[0m"); } while(0)
+ fprintf(stderr, "\033[0m"); } while (0)
#define errorPrint(fmt, ...) \
do { fprintf(stderr, "\033[31m"); \
fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \
- fprintf(stderr, "\033[0m"); } while(0)
+ fprintf(stderr, "\033[0m"); } while (0)
#define okPrint(fmt, ...) \
do { fprintf(stderr, "\033[32m"); \
fprintf(stderr, "OK: "fmt, __VA_ARGS__); \
- fprintf(stderr, "\033[0m"); } while(0)
+ fprintf(stderr, "\033[0m"); } while (0)
int64_t g_num_of_tb = 2;
int64_t g_num_of_rec = 3;
@@ -74,10 +86,16 @@ static void prepare_data(TAOS* taos) {
res = taos_query(taos, "create database test;");
taos_free_result(res);
taosMsleep(100);
- taos_select_db(taos, "test");
+ if (taos_select_db(taos, "test")) {
+ errorPrint("%s() LN%d: taos_select_db() failed\n",
+ __func__, __LINE__);
+ return;
+ }
char command[1024] = {0};
- sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));");
+ sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, "
+ "bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, "
+ "city binary(20), dist nchar(20), street binary(20));");
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
okPrint("%s created\n", "meters");
@@ -117,12 +135,11 @@ static void prepare_data(TAOS* taos) {
sprintf(command, "insert into t%"PRId64" "
"values(%" PRId64 ", %f, %"PRId64", "
"'%c%d', '%s%c%d', '%c%d')",
- i, 1650000000000+j, (float)j, j,
+ i, 1650000000000+j, j * 1.0, j,
'a'+(int)j%25, rand(),
// "涛思", 'z' - (int)j%25, rand(),
"TAOS", 'z' - (int)j%25, rand(),
- 'b' - (int)j%25, rand()
- );
+ 'b' - (int)j%25, rand());
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
affected = taos_affected_rows(res);
@@ -150,8 +167,7 @@ static void prepare_data(TAOS* taos) {
i, 1650000000000+j+1, (float)j, j,
'a'+(int)j%25, rand(),
"数据", 'z' - (int)j%25, rand(),
- 'b' - (int)j%25, rand()
- );
+ 'b' - (int)j%25, rand());
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
affected = taos_affected_rows(res);
@@ -179,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
}
if (block) {
- warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
+ warnPrint("%s", "call taos_fetch_block(), "
+ "don't call taos_fetch_lengths()\n");
int rows = 0;
while ((rows = taos_fetch_block(res, &row))) {
int *lengths = taos_fetch_lengths(res);
@@ -190,7 +207,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d type is %d, no need get offset\n",
f, fields[f].type);
for (int64_t c = 0; c < rows; c++) {
- switch(fields[f].type) {
+ switch (fields[f].type) {
case TSDB_DATA_TYPE_TIMESTAMP:
if (taos_is_null(res, c, f)) {
printf("col%d, row: %"PRId64" "
@@ -199,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %"PRId64"\n",
f, c,
- *(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
+ *(int64_t*)((char*)(row[f])
+ +c*sizeof(int64_t)));
}
break;
@@ -211,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %d\n",
f, c,
- *(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
+ *(int32_t*)((char*)(row[f])
+ +c*sizeof(int32_t)));
}
break;
@@ -223,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %f\n",
f, c,
- *(float*)((char*)(row[f])+c*sizeof(float)));
+ *(float*)((char*)(row[f])
+ +c*sizeof(float)));
}
break;
@@ -238,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
if (offsets) {
for (int c = 0; c < rows; c++) {
if (offsets[c] != -1) {
- int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
+ int length = *(int16_t*)((char*)(row[f])
+ + offsets[c]);
char *buf = calloc(1, length + 1);
- strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
- printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
+ strncpy(buf, (char *)((char*)(row[f])
+ + offsets[c] + 2), length);
+ printf("row: %d, col: %d, offset: %d, "
+ "length: %d, content: %s\n",
c, f, offsets[c], length, buf);
free(buf);
} else {
- printf("row: %d, col: %d, offset: -1, means content is NULL\n",
+ printf("row: %d, col: %d, offset: -1, "
+ "means content is NULL\n",
c, f);
}
}
@@ -267,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
int* lengths = taos_fetch_lengths(res);
if (lengths) {
for (int c = 0; c < num_fields; c++) {
- printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n",
+ printf("row: %"PRId64", col: %d, is_null: %s, "
+ "length of column %d is %d\n",
num_rows, c,
taos_is_null(res, num_rows, c)?"True":"False",
c, lengths[c]);
@@ -277,7 +302,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
__func__, __LINE__, tbname);
}
- num_rows ++;
+ num_rows++;
}
}
@@ -322,7 +347,8 @@ int main(int argc, char *argv[]) {
#endif
TAOS* taos = taos_connect(host, user, passwd, "", 0);
if (taos == NULL) {
- printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));
+ printf("\033[31mfailed to connect to db, reason:%s\033[0m\n",
+ taos_errstr(taos));
exit(1);
}
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 0e22985059..fd0521af3b 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -93,6 +93,7 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in
// query client
extern int32_t tsQueryPolicy;
+extern int32_t tsQueryRspPolicy;
extern int32_t tsQuerySmaOptimize;
extern int32_t tsQueryRsmaTolerance;
extern bool tsQueryPlannerTrace;
diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h
index 0d43539629..cdb1642a5c 100644
--- a/include/dnode/mnode/mnode.h
+++ b/include/dnode/mnode/mnode.h
@@ -30,9 +30,10 @@ typedef struct SMnode SMnode;
typedef struct {
int32_t dnodeId;
- bool standby;
bool deploy;
- SReplica replica;
+ int8_t selfIndex;
+ int8_t numOfReplicas;
+ SReplica replicas[TSDB_MAX_REPLICA];
SMsgCb msgCb;
} SMnodeOpt;
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index 777fe1c4df..7d95e840e9 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -54,6 +54,9 @@ typedef enum {
#define QUERY_POLICY_QNODE 3
#define QUERY_POLICY_CLIENT 4
+#define QUERY_RSP_POLICY_DELAY 0
+#define QUERY_RSP_POLICY_QUICK 1
+
typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema
uint8_t precision; // the number of precision
diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h
index 411a63a379..0f8220f19c 100644
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@@ -211,7 +211,7 @@ typedef struct SSyncInfo {
int32_t syncInit();
void syncCleanUp();
-int64_t syncOpen(const SSyncInfo* pSyncInfo);
+int64_t syncOpen(SSyncInfo* pSyncInfo);
void syncStart(int64_t rid);
void syncStop(int64_t rid);
int32_t syncSetStandby(int64_t rid);
@@ -233,7 +233,7 @@ const char* syncStr(ESyncState state);
bool syncIsRestoreFinish(int64_t rid);
int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot);
-int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg);
+int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg);
// build SRpcMsg, need to call syncPropose with SRpcMsg
int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg);
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index de31a970df..adf244e32a 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -43,6 +43,7 @@ extern "C" {
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL
#define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3)
+#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE)
typedef enum {
TAOS_WAL_WRITE = 1,
diff --git a/include/os/osEnv.h b/include/os/osEnv.h
index 293d9d17f8..d4e94d6173 100644
--- a/include/os/osEnv.h
+++ b/include/os/osEnv.h
@@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace;
void osDefaultInit();
void osUpdate();
void osCleanup();
+
bool osLogSpaceAvailable();
bool osDataSpaceAvailable();
bool osTempSpaceAvailable();
+
+bool osLogSpaceSufficient();
+bool osDataSpaceSufficient();
+bool osTempSpaceSufficient();
+
void osSetTimezone(const char *timezone);
void osSetSystemLocale(const char *inLocale, const char *inCharSet);
@@ -59,4 +65,4 @@ void osSetSystemLocale(const char *inLocale, const char *inCharSet);
}
#endif
-#endif /*_TD_OS_ENV_H_*/
\ No newline at end of file
+#endif /*_TD_OS_ENV_H_*/
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 6bc0e0e7dd..bbddb539c6 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -450,6 +450,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003)
#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004)
#define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005)
+#define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006)
// tfs
#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201)
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 7b437e4769..940dd745c8 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1036,30 +1036,38 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
pRequest->type = pQuery->msgType;
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
+
SPlanContext cxt = {.queryId = pRequest->requestId,
- .acctId = pRequest->pTscObj->acctId,
- .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
- .pAstRoot = pQuery->pRoot,
- .showRewrite = pQuery->showRewrite,
- .pMsg = pRequest->msgBuf,
- .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
- .pUser = pRequest->pTscObj->user,
- .sysInfo = pRequest->pTscObj->sysInfo,
- .allocatorId = pRequest->allocatorRefId};
- SQueryPlan* pDag = NULL;
- int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
- if (code) {
- tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
- pRequest->requestId);
- } else {
- pRequest->body.subplanNum = pDag->numOfSubplans;
+ .acctId = pRequest->pTscObj->acctId,
+ .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
+ .pAstRoot = pQuery->pRoot,
+ .showRewrite = pQuery->showRewrite,
+ .pMsg = pRequest->msgBuf,
+ .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+ .pUser = pRequest->pTscObj->user,
+ .sysInfo = pRequest->pTscObj->sysInfo,
+ .allocatorId = pRequest->allocatorRefId};
+
+ SAppInstInfo* pAppInfo = getAppInfo(pRequest);
+ SQueryPlan* pDag = NULL;
+
+ int64_t st = taosGetTimestampUs();
+ int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
+ if (code) {
+ tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
+ pRequest->requestId);
+ } else {
+ pRequest->body.subplanNum = pDag->numOfSubplans;
+ }
+
+ pRequest->metric.planEnd = taosGetTimestampUs();
+ if (code == TSDB_CODE_SUCCESS) {
+ tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self,
+ (pRequest->metric.planEnd - st)/1000.0, pRequest->requestId);
}
-
- pRequest->metric.planEnd = taosGetTimestampUs();
-
- if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
- SArray* pNodeList = NULL;
- buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
+ if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
+ SArray* pNodeList = NULL;
+ buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter,
.requestId = pRequest->requestId,
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 6f8cef7c0d..6126817ece 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -146,7 +146,7 @@ void taos_close(TAOS *taos) {
int taos_errno(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
- if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
return terrno;
}
@@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) {
return 0;
}
- return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL
- : ((SRequestObj *)res)->code;
+ return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code;
}
const char *taos_errstr(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
- if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
return (const char *)tstrerror(terrno);
}
@@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) {
if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
return pRequest->msgBuf;
} else {
- return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL)
+ return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY)
: (const char *)tstrerror(pRequest->code);
}
}
@@ -222,7 +221,9 @@ void taos_kill_query(TAOS *taos) {
int64_t rid = *(int64_t *)taos;
STscObj *pTscObj = acquireTscObj(rid);
- stopAllRequests(pTscObj->pRequests);
+ if (pTscObj) {
+ stopAllRequests(pTscObj->pRequests);
+ }
releaseTscObj(rid);
}
diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index e69174476a..21a52a4b57 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) {
int32_t n = 0;
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
- int32_t* length = taos_fetch_lengths(pRes);
- for (int32_t i = 0; i < numOfFields; ++i) {
- printf("(%d):%d ", i, length[i]);
- }
- printf("\n");
-
- int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
- printf("%s\n", str);
- memset(str, 0, sizeof(str));
+// int32_t* length = taos_fetch_lengths(pRes);
+// for(int32_t i = 0; i < numOfFields; ++i) {
+// printf("(%d):%d " , i, length[i]);
+// }
+// printf("\n");
+//
+// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+// printf("%s\n", str);
+// memset(str, 0, sizeof(str));
}
}
@@ -102,17 +102,6 @@ void queryCallback(void* param, void* res, int32_t code) {
taos_fetch_raw_block_a(res, fetchCallback, param);
}
-void queryCallback1(void* param, void* res, int32_t code) {
- if (code != TSDB_CODE_SUCCESS) {
- printf("failed to execute, reason:%s\n", taos_errstr(res));
- }
-
- taos_free_result(res);
-
- printf("exec query:\n");
- taos_query_a(param, "select * from tm1", queryCallback, param);
-}
-
void createNewTable(TAOS* pConn, int32_t index) {
char str[1024] = {0};
sprintf(str, "create table tu%d using st2 tags(%d)", index, index);
@@ -123,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
}
taos_free_result(pRes);
- for (int32_t i = 0; i < 3280; i += 20) {
+ for(int32_t i = 0; i < 10000; i += 20) {
char sql[1024] = {0};
sprintf(sql,
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
@@ -141,10 +130,49 @@ void createNewTable(TAOS* pConn, int32_t index) {
taos_free_result(p);
}
}
+
+void *queryThread(void *arg) {
+ TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0);
+ if (pConn == NULL) {
+ printf("failed to connect to db, reason:%s", taos_errstr(pConn));
+ return NULL;
+ }
+
+ int64_t el = 0;
+
+ for (int32_t i = 0; i < 5000000; ++i) {
+ int64_t st = taosGetTimestampUs();
+ TAOS_RES* pRes = taos_query(pConn,
+ "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;");
+ if (taos_errno(pRes) != 0) {
+ printf("failed, reason:%s\n", taos_errstr(pRes));
+ } else {
+ printResult(pRes);
+ }
+
+ taos_free_result(pRes);
+ el += (taosGetTimestampUs() - st);
+ if (i % 1000 == 0 && i != 0) {
+ printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000));
+ }
+ }
+
+ taos_close(pConn);
+ return NULL;
+}
+
+static int32_t numOfThreads = 1;
} // namespace
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
+ if (argc > 1) {
+ numOfThreads = atoi(argv[1]);
+ }
+
+ numOfThreads = TMAX(numOfThreads, 1);
+ printf("the runing threads is:%d", numOfThreads);
+
return RUN_ALL_TESTS();
}
@@ -664,7 +692,6 @@ TEST(testCase, insert_test) {
taos_free_result(pRes);
taos_close(pConn);
}
-#endif
TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -676,13 +703,14 @@ TEST(testCase, projection_query_tables) {
// }
// taos_free_result(pRes);
- TAOS_RES* pRes = taos_query(pConn, "use abc1");
+ TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu");
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
}
+
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)");
@@ -722,6 +750,16 @@ TEST(testCase, projection_query_tables) {
taos_free_result(pRes);
taos_close(pConn);
}
+#endif
+
+TEST(testCase, tsbs_perf_test) {
+ TdThread qid[20] = {0};
+
+ for(int32_t i = 0; i < numOfThreads; ++i) {
+ taosThreadCreate(&qid[i], NULL, queryThread, NULL);
+ }
+ getchar();
+}
#if 0
TEST(testCase, projection_query_stables) {
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 88df54a9ea..56daa4075a 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -90,6 +90,7 @@ bool tsSmlDataFormat = false;
// query
int32_t tsQueryPolicy = 1;
+int32_t tsQueryRspPolicy = 0;
int32_t tsQuerySmaOptimize = 0;
int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data.
bool tsQueryPlannerTrace = false;
@@ -350,6 +351,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1;
if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1;
+ if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1;
@@ -728,6 +730,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32;
tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
+ tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32;
tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32;
diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
index a4c37dd334..17cbde437b 100644
--- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
+++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h
@@ -41,8 +41,8 @@ typedef struct SMnodeMgmt {
} SMnodeMgmt;
// mmFile.c
-int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed);
-int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed);
+int32_t mmReadFile(const char *path, SMnodeOpt *pOption);
+int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption);
// mmHandle.c
SArray *mmGetMsgHandles();
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
index 27a35ae17a..c5ddb9f021 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c
@@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "mmInt.h"
-int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
+int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
int32_t code = TSDB_CODE_INVALID_JSON_FORMAT;
int32_t len = 0;
int32_t maxLen = 4096;
@@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
char file[PATH_MAX] = {0};
TdFilePtr pFile = NULL;
- snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+ snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
code = 0;
@@ -50,38 +50,69 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) {
dError("failed to read %s since deployed not found", file);
goto _OVER;
}
- *pDeployed = deployed->valueint;
+ pOption->deploy = deployed->valueint;
- cJSON *id = cJSON_GetObjectItem(root, "id");
- if (id) {
- if (id->type != cJSON_Number) {
- dError("failed to read %s since id not found", file);
+ cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex");
+ if (selfIndex) {
+ if (selfIndex->type != cJSON_Number) {
+ dError("failed to read %s since selfIndex not found", file);
goto _OVER;
}
- if (pReplica) {
- pReplica->id = id->valueint;
- }
+ pOption->selfIndex = selfIndex->valueint;
}
- cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn");
- if (fqdn) {
- if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
- dError("failed to read %s since fqdn not found", file);
+ cJSON *replicas = cJSON_GetObjectItem(root, "replicas");
+ if (replicas) {
+ if (replicas->type != cJSON_Array) {
+ dError("failed to read %s since replicas not found", file);
goto _OVER;
}
- if (pReplica) {
- tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
- }
- }
- cJSON *port = cJSON_GetObjectItem(root, "port");
- if (port) {
- if (port->type != cJSON_Number) {
- dError("failed to read %s since port not found", file);
+ int32_t numOfReplicas = cJSON_GetArraySize(replicas);
+ if (numOfReplicas <= 0) {
+ dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas);
goto _OVER;
}
- if (pReplica) {
- pReplica->port = (uint16_t)port->valueint;
+ pOption->numOfReplicas = numOfReplicas;
+
+ for (int32_t i = 0; i < numOfReplicas; ++i) {
+ SReplica *pReplica = pOption->replicas + i;
+
+ cJSON *replica = cJSON_GetArrayItem(replicas, i);
+ if (replica == NULL) break;
+
+ cJSON *id = cJSON_GetObjectItem(replica, "id");
+ if (id) {
+ if (id->type != cJSON_Number) {
+ dError("failed to read %s since id not found", file);
+ goto _OVER;
+ }
+ if (pReplica) {
+ pReplica->id = id->valueint;
+ }
+ }
+
+ cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn");
+ if (fqdn) {
+ if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
+ dError("failed to read %s since fqdn not found", file);
+ goto _OVER;
+ }
+ if (pReplica) {
+ tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
+ }
+ }
+
+ cJSON *port = cJSON_GetObjectItem(replica, "port");
+ if (port) {
+ if (port->type != cJSON_Number) {
+ dError("failed to read %s since port not found", file);
+ goto _OVER;
+ }
+ if (pReplica) {
+ pReplica->port = (uint16_t)port->valueint;
+ }
+ }
}
}
@@ -92,18 +123,18 @@ _OVER:
if (root != NULL) cJSON_Delete(root);
if (pFile != NULL) taosCloseFile(&pFile);
if (code == 0) {
- dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+ dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy);
}
terrno = code;
return code;
}
-int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) {
+int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
- snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
- snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+ snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
+ snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
@@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
char *content = taosMemoryCalloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
- if (pReplica != NULL && pReplica->id > 0) {
- len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id);
- len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn);
- len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port);
+ if (pOption->deploy && pOption->numOfReplicas > 0) {
+ len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex);
+ len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n");
+
+ for (int32_t i = 0; i < pOption->numOfReplicas; ++i) {
+ const SReplica *pReplica = pOption->replicas + i;
+ if (pReplica != NULL && pReplica->id > 0) {
+ len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id);
+ len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn);
+ len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port);
+ }
+ if (i < pOption->numOfReplicas - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }],\n");
+ }
+ }
}
- len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed);
+ len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy);
len += snprintf(content + len, maxLen - len, "}\n");
taosWriteFile(pFile, content, len);
@@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed)
return -1;
}
- dDebug("successed to write %s, deployed:%d", realfile, deployed);
+ dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy);
return 0;
}
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
index ec761e6441..fc45dbf15f 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
return -1;
}
- if (createReq.replica != 1) {
+ SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1};
+ memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas));
+ for (int32_t i = 0; i < option.numOfReplicas; ++i) {
+ if (createReq.replicas[i].id == pInput->pData->dnodeId) {
+ option.selfIndex = i;
+ }
+ }
+
+ if (option.selfIndex == -1) {
terrno = TSDB_CODE_INVALID_OPTION;
- dGError("failed to create mnode since %s", terrstr());
+ dGError("failed to create mnode since %s, selfIndex is -1", terrstr());
return -1;
}
- bool deployed = true;
-
- SMnodeMgmt mgmt = {0};
- mgmt.path = pInput->path;
- mgmt.name = pInput->name;
- if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) {
+ if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
}
@@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
return -1;
}
- bool deployed = false;
-
- SMnodeMgmt mgmt = {0};
- mgmt.path = pInput->path;
- mgmt.name = pInput->name;
- if (mmWriteFile(&mgmt, NULL, deployed) != 0) {
+ SMnodeOpt option = {.deploy = false};
+ if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
}
@@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
- // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c
index 49207225a5..e7f71ad420 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c
@@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
}
static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
- SMnodeMgmt mgmt = {0};
- mgmt.path = pInput->path;
- if (mmReadFile(&mgmt, NULL, required) != 0) {
+ SMnodeOpt option = {0};
+ if (mmReadFile(pInput->path, &option) != 0) {
return -1;
}
- if (!(*required)) {
+ if (!option.deploy) {
*required = mmDeployRequired(pInput);
+ } else {
+ *required = true;
}
return 0;
}
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
- pOption->standby = false;
pOption->deploy = true;
pOption->msgCb = pMgmt->msgCb;
pOption->dnodeId = pMgmt->pData->dnodeId;
- pOption->replica.id = 1;
- pOption->replica.port = tsServerPort;
- tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
+ pOption->selfIndex = 0;
+ pOption->numOfReplicas = 1;
+ pOption->replicas[0].id = 1;
+ pOption->replicas[0].port = tsServerPort;
+ tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
}
-static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) {
- pOption->standby = false;
+static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
pOption->deploy = false;
pOption->msgCb = pMgmt->msgCb;
pOption->dnodeId = pMgmt->pData->dnodeId;
- if (pReplica->id > 0) {
- pOption->standby = true;
- pOption->replica = *pReplica;
- }
}
static void mmClose(SMnodeMgmt *pMgmt) {
@@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
- bool deployed = false;
- SReplica replica = {0};
- if (mmReadFile(pMgmt, &replica, &deployed) != 0) {
+ SMnodeOpt option = {0};
+ if (mmReadFile(pMgmt->path, &option) != 0) {
dError("failed to read file since %s", terrstr());
mmClose(pMgmt);
return -1;
}
- SMnodeOpt option = {0};
- if (!deployed) {
+ if (!option.deploy) {
dInfo("mnode start to deploy");
pMgmt->pData->dnodeId = 1;
mmBuildOptionForDeploy(pMgmt, pInput, &option);
} else {
dInfo("mnode start to open");
- mmBuildOptionForOpen(pMgmt, &replica, &option);
+ mmBuildOptionForOpen(pMgmt, &option);
}
pMgmt->pMnode = mndOpen(pMgmt->path, &option);
@@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
}
tmsgReportStartup("mnode-worker", "initialized");
- if (!deployed || replica.id > 0) {
- deployed = true;
- if (mmWriteFile(pMgmt, NULL, deployed) != 0) {
+ if (option.numOfReplicas > 0) {
+ option.deploy = true;
+ option.numOfReplicas = 0;
+ if (mmWriteFile(pMgmt->path, &option) != 0) {
dError("failed to write mnode file since %s", terrstr());
return -1;
}
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
index d2b071ec61..849d3ef390 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
@@ -72,7 +72,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
mndPostProcessQueryMsg(pMsg);
}
- dGTrace("msg:%p, is freed, code:0x%x", pMsg, code);
+ dGTrace("msg:%p is freed, code:%s", pMsg, tstrerror(code));
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
index d07ec7abb0..076826ebc2 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
@@ -51,26 +51,14 @@ static int32_t dmInitMonitor() {
static bool dmCheckDiskSpace() {
osUpdate();
- if (!osDataSpaceAvailable()) {
- dError("free disk size: %f GB, too little, require %f GB at least at least , quit",
- (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
- (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
- terrno = TSDB_CODE_NO_AVAIL_DISK;
- return false;
+ if (!osDataSpaceSufficient()) {
+ dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0);
}
- if (!osLogSpaceAvailable()) {
- dError("free disk size: %f GB, too little, require %f GB at least at least, quit",
- (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
- (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
- terrno = TSDB_CODE_NO_AVAIL_DISK;
- return false;
+ if (!osLogSpaceSufficient()) {
+ dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0);
}
- if (!osTempSpaceAvailable()) {
- dError("free disk size: %f GB, too little, require %f GB at least at least, quit",
- (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0,
- (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
- terrno = TSDB_CODE_NO_AVAIL_DISK;
- return false;
+ if (!osTempSpaceSufficient()) {
+ dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0);
}
return true;
}
diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h
index 71315a39fd..f55e830a44 100644
--- a/source/dnode/mnode/impl/inc/mndInt.h
+++ b/source/dnode/mnode/impl/inc/mndInt.h
@@ -87,12 +87,13 @@ typedef struct {
typedef struct {
tsem_t syncSem;
int64_t sync;
- SReplica replica;
int32_t errCode;
int32_t transId;
SRWLatch lock;
- int8_t standby;
int8_t leaderTransferFinish;
+ int8_t selfIndex;
+ int8_t numOfReplicas;
+ SReplica replicas[TSDB_MAX_REPLICA];
} SSyncMgmt;
typedef struct {
@@ -130,11 +131,10 @@ typedef struct SMnode {
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
int64_t mndGenerateUid(const char *name, int32_t len);
-int32_t mndAcquireRpcRef(SMnode *pMnode);
-void mndReleaseRpcRef(SMnode *pMnode);
-void mndSetRestore(SMnode *pMnode, bool restored);
-void mndSetStop(SMnode *pMnode);
-bool mndGetStop(SMnode *pMnode);
+void mndSetRestored(SMnode *pMnode, bool restored);
+bool mndGetRestored(SMnode *pMnode);
+void mndSetStop(SMnode *pMnode);
+bool mndGetStop(SMnode *pMnode);
#ifdef __cplusplus
}
diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h
index cb9d70d5ee..993efbcd08 100644
--- a/source/dnode/mnode/impl/inc/mndSync.h
+++ b/source/dnode/mnode/impl/inc/mndSync.h
@@ -24,7 +24,7 @@ extern "C" {
int32_t mndInitSync(SMnode *pMnode);
void mndCleanupSync(SMnode *pMnode);
-bool mndIsMaster(SMnode *pMnode);
+bool mndIsLeader(SMnode *pMnode);
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId);
void mndSyncStart(SMnode *pMnode);
void mndSyncStop(SMnode *pMnode);
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 6670211d78..3b31873857 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
}
while (numOfRows < rowsCapacity) {
- pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus);
+ pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true);
if (pShow->pIter == NULL) break;
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 9ade39e8e7..fba1fd94d6 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
goto _OVER;
} else {
pDnode->accessTimes++;
- mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
+ mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
}
}
@@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
}
pDnode->lastAccessTime = curMs;
+ pDnode->accessTimes++;
code = 0;
_OVER:
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index 275fc76e36..5fb23b045c 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -43,6 +43,37 @@
#include "mndUser.h"
#include "mndVgroup.h"
+static inline int32_t mndAcquireRpc(SMnode *pMnode) {
+ int32_t code = 0;
+ taosThreadRwlockRdlock(&pMnode->lock);
+ if (pMnode->stopped) {
+ terrno = TSDB_CODE_APP_NOT_READY;
+ code = -1;
+ } else if (!mndIsLeader(pMnode)) {
+ code = -1;
+ } else {
+#if 1
+ atomic_add_fetch_32(&pMnode->rpcRef, 1);
+#else
+ int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
+ mTrace("mnode rpc is acquired, ref:%d", ref);
+#endif
+ }
+ taosThreadRwlockUnlock(&pMnode->lock);
+ return code;
+}
+
+static inline void mndReleaseRpc(SMnode *pMnode) {
+ taosThreadRwlockRdlock(&pMnode->lock);
+#if 1
+ atomic_sub_fetch_32(&pMnode->rpcRef, 1);
+#else
+ int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
+ mTrace("mnode rpc is released, ref:%d", ref);
+#endif
+ taosThreadRwlockUnlock(&pMnode->lock);
+}
+
static void *mndBuildTimerMsg(int32_t *pContLen) {
SMTimerReq timerReq = {0};
@@ -201,7 +232,7 @@ static int32_t mndInitWal(SMnode *pMnode) {
pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) {
- mError("failed to open wal since %s", terrstr());
+ mError("failed to open wal since %s. wal:%s", terrstr(), path);
return -1;
}
@@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) {
static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
pMnode->msgCb = pOption->msgCb;
pMnode->selfDnodeId = pOption->dnodeId;
- pMnode->syncMgmt.replica = pOption->replica;
- pMnode->syncMgmt.standby = pOption->standby;
+ pMnode->syncMgmt.selfIndex = pOption->selfIndex;
+ pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas;
+ memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas));
}
SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
@@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) {
mError("failed to deploy sdb while start mnode");
return -1;
}
- mndSetRestore(pMnode, true);
+ mndSetRestored(pMnode, true);
}
grantReset(pMnode, TSDB_GRANT_ALL, 0);
@@ -570,23 +602,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) {
return 0;
}
- if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
+ if (mndAcquireRpc(pMsg->info.node) == 0) return 0;
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
pMsg->msgType == TDMT_MND_UPTIME_TIMER) {
return -1;
}
- SEpSet epSet = {0};
- mndGetMnodeEpSet(pMsg->info.node, &epSet);
+ SEpSet epSet = {0};
+ SMnode *pMnode = pMsg->info.node;
+ mndGetMnodeEpSet(pMnode, &epSet);
const STraceId *trace = &pMsg->info.traceId;
- mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(),
- TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
+ mDebug(
+ "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s "
+ "numOfEps:%d inUse:%d",
+ pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync),
+ syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
if (epSet.numOfEps > 0) {
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
- mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
+ mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
}
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
@@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
int32_t code = (*fp)(pMsg);
- mndReleaseRpcRef(pMnode);
+ mndReleaseRpc(pMnode);
if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
mGTrace("msg:%p, won't response immediately since in progress", pMsg);
@@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
- if (mndAcquireRpcRef(pMnode) != 0) return -1;
+ if (mndAcquireRpc(pMnode) != 0) return -1;
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
@@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) {
- mndReleaseRpcRef(pMnode);
+ mndReleaseRpc(pMnode);
return -1;
}
@@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pGrantInfo->timeseries_total = INT32_MAX;
}
- mndReleaseRpcRef(pMnode);
+ mndReleaseRpc(pMnode);
return 0;
}
@@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
return 0;
}
-int32_t mndAcquireRpcRef(SMnode *pMnode) {
- int32_t code = 0;
- taosThreadRwlockRdlock(&pMnode->lock);
- if (pMnode->stopped) {
- mTrace("mnode not running");
- terrno = TSDB_CODE_APP_NOT_READY;
- code = -1;
- } else if (!mndIsMaster(pMnode)) {
- mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored);
- code = -1;
- } else {
- int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
- // mTrace("mnode rpc is acquired, ref:%d", ref);
- }
- taosThreadRwlockUnlock(&pMnode->lock);
- return code;
-}
-
-void mndReleaseRpcRef(SMnode *pMnode) {
- taosThreadRwlockRdlock(&pMnode->lock);
- int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1);
- // mTrace("mnode rpc is released, ref:%d", ref);
- taosThreadRwlockUnlock(&pMnode->lock);
-}
-
-void mndSetRestore(SMnode *pMnode, bool restored) {
+void mndSetRestored(SMnode *pMnode, bool restored) {
if (restored) {
taosThreadRwlockWrlock(&pMnode->lock);
pMnode->restored = true;
diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c
index d0440359ff..c27241317c 100644
--- a/source/dnode/mnode/impl/src/mndMnode.c
+++ b/source/dnode/mnode/impl/src/mndMnode.c
@@ -36,6 +36,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq);
static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq);
static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter);
+static void mndReloadSyncConfig(SMnode *pMnode);
int32_t mndInitMnode(SMnode *pMnode) {
SSdbTable table = {
@@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
}
pObj->state = TAOS_SYNC_STATE_ERROR;
+ mndReloadSyncConfig(pSdb->pMnode);
return 0;
}
@@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) {
mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
pOld->updateTime = pNew->updateTime;
+ mndReloadSyncConfig(pSdb->pMnode);
+
return 0;
}
@@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (pIter == NULL) break;
if (pObj->id == pMnode->selfDnodeId) {
- if (mndIsMaster(pMnode)) {
+ if (mndIsLeader(pMnode)) {
pEpSet->inUse = pEpSet->numOfEps;
} else {
pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes;
@@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (pEpSet->numOfEps == 0) {
syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet);
}
+
+ if (pEpSet->inUse >= pEpSet->numOfEps) {
+ pEpSet->inUse = 0;
+ }
}
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
@@ -274,13 +282,72 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod
return 0;
}
+static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
+ int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
+ void *pReq = taosMemoryMalloc(contLen);
+ tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
+
+ STransAction action = {
+ .epSet = *pCreateEpSet,
+ .pCont = pReq,
+ .contLen = contLen,
+ .msgType = TDMT_DND_CREATE_MNODE,
+ .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED,
+ };
+
+ if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+ taosMemoryFree(pReq);
+ return -1;
+ }
+ return 0;
+}
+
+static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
+ int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
+ void *pReq = taosMemoryMalloc(contLen);
+ tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
+
+ STransAction action = {
+ .epSet = *pAlterEpSet,
+ .pCont = pReq,
+ .contLen = contLen,
+ .msgType = TDMT_MND_ALTER_MNODE,
+ .acceptableCode = 0,
+ };
+
+ if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+ taosMemoryFree(pReq);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
+ int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq);
+ void *pReq = taosMemoryMalloc(contLen);
+ tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq);
+
+ STransAction action = {
+ .epSet = *pDroprEpSet,
+ .pCont = pReq,
+ .contLen = contLen,
+ .msgType = TDMT_DND_DROP_MNODE,
+ .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
+ };
+
+ if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+ taosMemoryFree(pReq);
+ return -1;
+ }
+ return 0;
+}
+
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int32_t numOfReplicas = 0;
- SDAlterMnodeReq alterReq = {0};
SDCreateMnodeReq createReq = {0};
- SEpSet alterEpset = {0};
SEpSet createEpset = {0};
while (1) {
@@ -288,75 +355,25 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
- alterReq.replicas[numOfReplicas].id = pMObj->id;
- alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-
- alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
- if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
- alterEpset.inUse = numOfReplicas;
- }
+ createReq.replicas[numOfReplicas].id = pMObj->id;
+ createReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
+ memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
numOfReplicas++;
sdbRelease(pSdb, pMObj);
}
- alterReq.replica = numOfReplicas + 1;
- alterReq.replicas[numOfReplicas].id = pDnode->id;
- alterReq.replicas[numOfReplicas].port = pDnode->port;
- memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-
- alterEpset.numOfEps = numOfReplicas + 1;
- alterEpset.eps[numOfReplicas].port = pDnode->port;
- memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-
- createReq.replica = 1;
- createReq.replicas[0].id = pDnode->id;
- createReq.replicas[0].port = pDnode->port;
- memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
+ createReq.replica = numOfReplicas + 1;
+ createReq.replicas[numOfReplicas].id = pDnode->id;
+ createReq.replicas[numOfReplicas].port = pDnode->port;
+ memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
+ createEpset.inUse = 0;
createEpset.numOfEps = 1;
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
- {
- int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSDCreateMnodeReq(pReq, contLen, &createReq);
-
- STransAction action = {
- .epSet = createEpset,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_DND_CREATE_MNODE,
- .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
-
- {
- int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
-
- STransAction action = {
- .epSet = alterEpset,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_MND_ALTER_MNODE,
- .acceptableCode = 0,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
+ if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
return 0;
}
@@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
+ if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
- if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndTransAppendNullLog(pTrans) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -459,107 +476,28 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO
}
static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
- SSdb *pSdb = pMnode->pSdb;
- void *pIter = NULL;
- int32_t numOfReplicas = 0;
- SDAlterMnodeReq alterReq = {0};
- SDDropMnodeReq dropReq = {0};
- SSetStandbyReq standbyReq = {0};
- SEpSet alterEpset = {0};
- SEpSet dropEpSet = {0};
-
- while (1) {
- SMnodeObj *pMObj = NULL;
- pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
- if (pIter == NULL) break;
- if (pMObj->id == pObj->id) {
- sdbRelease(pSdb, pMObj);
- continue;
- }
-
- alterReq.replicas[numOfReplicas].id = pMObj->id;
- alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
-
- alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port;
- memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
- if (pMObj->state == TAOS_SYNC_STATE_LEADER) {
- alterEpset.inUse = numOfReplicas;
- }
-
- numOfReplicas++;
- sdbRelease(pSdb, pMObj);
- }
-
- alterReq.replica = numOfReplicas;
- alterEpset.numOfEps = numOfReplicas;
+ SSdb *pSdb = pMnode->pSdb;
+ void *pIter = NULL;
+ int32_t numOfReplicas = 0;
+ SDDropMnodeReq dropReq = {0};
+ SEpSet dropEpSet = {0};
dropReq.dnodeId = pDnode->id;
dropEpSet.numOfEps = 1;
dropEpSet.eps[0].port = pDnode->port;
memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
- standbyReq.dnodeId = pDnode->id;
- standbyReq.standby = 1;
-
- {
- int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq);
- SMsgHead *pHead = pReq;
- pHead->contLen = htonl(contLen);
- pHead->vgId = htonl(MNODE_HANDLE);
-
- STransAction action = {
- .epSet = dropEpSet,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_SYNC_SET_MNODE_STANDBY,
- .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
-
- {
- int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq);
-
- STransAction action = {
- .epSet = alterEpset,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_MND_ALTER_MNODE,
- .acceptableCode = 0,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
- }
-
- {
- int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq);
- void *pReq = taosMemoryMalloc(contLen);
- tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq);
-
- STransAction action = {
- .epSet = dropEpSet,
- .pCont = pReq,
- .contLen = contLen,
- .msgType = TDMT_DND_DROP_MNODE,
- .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
- };
-
- if (mndTransAppendRedoAction(pTrans, &action) != 0) {
- taosMemoryFree(pReq);
- return -1;
- }
+ int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE);
+ if (totalMnodes == 2) {
+ mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
+ if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
+ if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+ } else if (totalMnodes == 3) {
+ mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
+ if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+ if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
+ } else {
+ return -1;
}
return 0;
@@ -567,7 +505,6 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
if (pObj == NULL) return 0;
- if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1;
if (mndTransAppendNullLog(pTrans) != 0) return -1;
@@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
int64_t curMs = taosGetTimestampMs();
while (numOfRows < rows) {
- pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus);
+ pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true);
if (pShow->pIter == NULL) break;
cols = 0;
@@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) {
}
static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
+#if 1
+ return 0;
+#else
SMnode *pMnode = pReq->info.node;
SDAlterMnodeReq alterReq = {0};
@@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
return -1;
}
+ SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
+ memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
+ for (int32_t i = 0; i < option.numOfReplicas; ++i) {
+ if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
+ option.selfIndex = i;
+ }
+ }
+
+ if (option.selfIndex == -1) {
+ mInfo("alter mnode not processed since selfIndex is -1", terrstr());
+ return 0;
+ }
+
+ if (mndWriteFile(pMnode->path, &option) != 0) {
+ mError("failed to write mnode file since %s", terrstr());
+ return -1;
+ }
+
SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
for (int32_t i = 0; i < alterReq.replica; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn));
pNode->nodePort = alterReq.replicas[i].port;
- if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i;
+ if (alterReq.replicas[i].id == pMnode->selfDnodeId) {
+ cfg.myIndex = i;
+ }
}
if (cfg.myIndex == -1) {
mError("failed to alter mnode since myindex is -1");
return -1;
} else {
- mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex);
+ mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
for (int32_t i = 0; i < alterReq.replica; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
}
}
- mInfo("trans:-1, sync reconfig will be proposed");
-
- SSyncMgmt *pMgmt = &pMnode->syncMgmt;
- pMgmt->standby = 0;
- int32_t code = syncReconfig(pMgmt->sync, &cfg);
+ int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) {
- mError("trans:-1, failed to propose sync reconfig since %s", terrstr());
- return code;
+ mError("failed to sync reconfig since %s", terrstr());
} else {
- pMgmt->errCode = 0;
- taosWLockLatch(&pMgmt->lock);
- pMgmt->transId = -1;
- taosWUnLockLatch(&pMgmt->lock);
- tsem_wait(&pMgmt->syncSem);
- mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode));
- terrno = pMgmt->errCode;
- return pMgmt->errCode;
+ mInfo("alter mnode sync success");
+ }
+
+ return code;
+#endif
+}
+
+static void mndReloadSyncConfig(SMnode *pMnode) {
+ SSdb *pSdb = pMnode->pSdb;
+ SMnodeObj *pObj = NULL;
+ ESdbStatus objStatus = 0;
+ void *pIter = NULL;
+ bool hasUpdatingMnode = false;
+ SSyncCfg cfg = {.myIndex = -1};
+
+ while (1) {
+ pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false);
+ if (pIter == NULL) break;
+ if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) {
+ mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus));
+ hasUpdatingMnode = true;
+ }
+
+ if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) {
+ SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum];
+ tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn));
+ pNode->nodePort = pObj->pDnode->port;
+ if (pObj->pDnode->id == pMnode->selfDnodeId) {
+ cfg.myIndex = cfg.replicaNum;
+ }
+ cfg.replicaNum++;
+ }
+
+ sdbReleaseLock(pSdb, pObj, false);
+ }
+
+ if (cfg.myIndex == -1) {
+ mInfo("vgId:1, mnode not reload since selfIndex is -1");
+ return;
+ }
+
+ if (!mndGetRestored(pMnode)) {
+ mInfo("vgId:1, mnode not reload since restore not finished");
+ return;
+ }
+
+ if (hasUpdatingMnode) {
+ mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
+ for (int32_t i = 0; i < cfg.replicaNum; ++i) {
+ SNodeInfo *pNode = &cfg.nodeInfo[i];
+ mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
+ }
+
+ int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
+ if (code != 0) {
+ mError("vgId:1, failed to reconfig mnode sync since %s", terrstr());
+ } else {
+ mInfo("vgId:1, reconfig mnode sync success");
+ }
}
}
diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c
index 0e4dd9fc3c..866a3fa8b9 100644
--- a/source/dnode/mnode/impl/src/mndSync.c
+++ b/source/dnode/mnode/impl/src/mndSync.c
@@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) {
if (!pMnode->deploy) {
mInfo("vgId:1, sync restore finished, and will handle outstanding transactions");
mndTransPullup(pMnode);
- mndSetRestore(pMnode, true);
+ mndSetRestored(pMnode, true);
} else {
mInfo("vgId:1, sync restore finished");
}
@@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) {
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP);
syncInfo.pWal = pMnode->pWal;
syncInfo.pFsm = mndSyncMakeFsm(pMnode);
- syncInfo.isStandBy = pMgmt->standby;
syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT;
- mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby);
- if (pMgmt->standby || pMgmt->replica.id > 0) {
- SSyncCfg *pCfg = &syncInfo.syncCfg;
- pCfg->replicaNum = 1;
- pCfg->myIndex = 0;
- SNodeInfo *pNode = &pCfg->nodeInfo[0];
- tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn));
- pNode->nodePort = pMgmt->replica.port;
- mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort);
+ mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas);
+ SSyncCfg *pCfg = &syncInfo.syncCfg;
+ pCfg->replicaNum = pMgmt->numOfReplicas;
+ pCfg->myIndex = pMgmt->selfIndex;
+ for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) {
+ SNodeInfo *pNode = &pCfg->nodeInfo[i];
+ tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
+ pNode->nodePort = pMgmt->replicas[i].port;
+ mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort);
}
tsem_init(&pMgmt->syncSem, 0, 0);
@@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) {
setPingTimerMS(pMgmt->sync, 5000);
setElectTimerMS(pMgmt->sync, 3000);
setHeartbeatTimerMS(pMgmt->sync, 500);
- /*
- setElectTimerMS(pMgmt->sync, 600);
- setHeartbeatTimerMS(pMgmt->sync, 300);
- */
mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0;
@@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
syncSetMsgCb(pMgmt->sync, &pMnode->msgCb);
syncStart(pMgmt->sync);
- mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby);
+ mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync);
}
void mndSyncStop(SMnode *pMnode) {
@@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) {
taosWUnLockLatch(&pMnode->syncMgmt.lock);
}
-bool mndIsMaster(SMnode *pMnode) {
+bool mndIsLeader(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
if (!syncIsReady(pMgmt->sync)) {
@@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) {
return false;
}
- if (!pMnode->restored) {
+ if (!mndGetRestored(pMnode)) {
terrno = TSDB_CODE_APP_NOT_READY;
return false;
}
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index d30cc8f9ca..a64b07fdbd 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -53,7 +53,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
-static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
+static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); }
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransTimer(SRpcMsg *pReq);
diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h
index f922baf329..be93bd2f59 100644
--- a/source/dnode/mnode/sdb/inc/sdb.h
+++ b/source/dnode/mnode/sdb/inc/sdb.h
@@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey);
* @param pObj The object of the row.
*/
void sdbRelease(SSdb *pSdb, void *pObj);
+void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock);
/**
* @brief Traverse a sdb table
@@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj);
* @return void* The next iterator of the table.
*/
void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj);
-void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status);
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock);
/**
* @brief Cancel a traversal
diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c
index ecdf8c71a7..0fee0fbca1 100644
--- a/source/dnode/mnode/sdb/src/sdbHash.c
+++ b/source/dnode/mnode/sdb/src/sdbHash.c
@@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) {
taosThreadRwlockUnlock(pLock);
}
-void sdbRelease(SSdb *pSdb, void *pObj) {
+void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) {
if (pObj == NULL) return;
SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow));
if (pRow->type >= SDB_MAX) return;
TdThreadRwlock *pLock = &pSdb->locks[pRow->type];
- taosThreadRwlockWrlock(pLock);
+ if (lock) {
+ taosThreadRwlockWrlock(pLock);
+ }
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "release");
@@ -342,9 +344,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
sdbFreeRow(pSdb, pRow, true);
}
- taosThreadRwlockUnlock(pLock);
+ if (lock) {
+ taosThreadRwlockUnlock(pLock);
+ }
}
+void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); }
+
void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
*ppObj = NULL;
@@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
return ppRow;
}
-void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) {
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) {
*ppObj = NULL;
SHashObj *hash = sdbGetHash(pSdb, type);
if (hash == NULL) return NULL;
TdThreadRwlock *pLock = &pSdb->locks[type];
- taosThreadRwlockRdlock(pLock);
+ if (lock) {
+ taosThreadRwlockRdlock(pLock);
+ }
SSdbRow **ppRow = taosHashIterate(hash, pIter);
while (ppRow != NULL) {
@@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat
*status = pRow->status;
break;
}
- taosThreadRwlockUnlock(pLock);
+ if (lock) {
+ taosThreadRwlockUnlock(pLock);
+ }
return ppRow;
}
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index c8a4ff945d..97b0be07c1 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -693,6 +693,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
if (iCol >= pSchema->nCols) break;
pColumn = &pSchema->pSchema[iCol];
+ ASSERT(pAlterTbReq->colName);
if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break;
iCol++;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 50c7a7cc53..0ee2e31097 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD
if (code) goto _err;
}
+_err:
if (aDelData) {
taosArrayDestroy(aDelData);
}
-_err:
return code;
}
@@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
// code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
- if (code) goto _err;
*pDelIdx = *pIdx;
+_err:
if (pDelIdxArray) {
taosArrayDestroy(pDelIdxArray);
}
-_err:
return code;
}
@@ -599,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
* &state->blockIdx);
*/
state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ);
- if (code) goto _err;
-
if (!state->pBlockIdx) {
goto _next_fileset;
}
@@ -898,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
if (code) goto _err;
code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
- if (code) goto _err;
+ if (code) {
+ tsdbDelFReaderClose(&pDelFReader);
+ goto _err;
+ }
code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline);
- if (code) goto _err;
+ if (code) {
+ tsdbDelFReaderClose(&pDelFReader);
+ goto _err;
+ }
tsdbDelFReaderClose(&pDelFReader);
} else {
@@ -1231,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
_err:
nextRowIterClose(&iter);
taosMemoryFreeClear(pTSchema);
+ *ppLastArray = NULL;
+ taosArrayDestroy(pColArray);
return code;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index aca9042261..a4d993bc6c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -1045,7 +1045,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
- ASSERT(eno == 0);
+ ASSERT(eno == 0 &&
+ "tsdbCommit failure"
+ "Restart taosd");
code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
TSDB_CHECK_CODE(code, lino, _exit);
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c
index da410532f9..d08b743763 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS.c
@@ -961,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname);
- taosRemoveFile(fname);
+ (void)taosRemoveFile(fname);
taosMemoryFree(pSetOld->pHeadF);
}
@@ -1114,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
- taosRemoveFile(fname);
+ (void)taosRemoveFile(fname);
taosMemoryFree(pSet->pHeadF);
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index 450032d4f2..0189ced3c6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
goto _err;
}
+ ASSERT(pPool != NULL);
// do delete
SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData));
if (pDelData == NULL) {
@@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
+ ASSERT(pPool != NULL);
pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
if (pTbData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN
// node
level = tsdbMemSkipListRandLevel(&pTbData->sl);
+ ASSERT(pPool != NULL);
pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
if (pNode == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 03d4a975f2..a4c5c60e78 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -133,8 +133,15 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
id.uid = pIter->uid;
}
- tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
+ code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
+
code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _exit;
+ }
double el = (taosGetTimestampUs() - st) / 1000.0;
pInfo->elapsedTime += el;
@@ -543,6 +550,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
pMTree->pLoadInfo = pBlockLoadInfo;
pMTree->destroyLoadInfo = destroyLoadInfo;
+ ASSERT(pMTree->pLoadInfo != NULL);
for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file
struct SLDataIter *pIter = NULL;
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index a7850f3114..c346eeef1d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2676,9 +2676,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
int8_t level = 0;
int8_t precision = pVnode->config.tsdbCfg.precision;
int64_t now = taosGetTimestamp(precision);
- int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1
- : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000
- : 1000000);
+ int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L
+ : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
+ : 1000000L);
for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
SRetention* pRetention = retentions + level;
@@ -3372,14 +3372,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
if (pCond->suid != 0) {
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1);
if (pReader->pSchema == NULL) {
- tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1,
- pReader->idStr);
+ tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr);
}
} else if (taosArrayGetSize(pTableList) > 0) {
STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1);
if (pReader->pSchema == NULL) {
- tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr);
+ tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);
}
}
@@ -3395,19 +3394,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
goto _err;
}
- code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr);
- if (code != TSDB_CODE_SUCCESS) {
- goto _err;
- }
-
- if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) {
- code = doOpenReaderImpl(pReader);
+ if (numOfTables > 0) {
+ code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
- return code;
+ goto _err;
}
- } else {
- STsdbReader* pPrevReader = pReader->innerReader[0];
- STsdbReader* pNextReader = pReader->innerReader[1];
+
+ if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) {
+ code = doOpenReaderImpl(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
+ } else {
+ STsdbReader* pPrevReader = pReader->innerReader[0];
+ STsdbReader* pNextReader = pReader->innerReader[1];
// we need only one row
pPrevReader->capacity = 1;
@@ -3422,19 +3422,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
pNextReader->pMemSchema = pReader->pMemSchema;
pNextReader->pReadSnap = pReader->pReadSnap;
- code = doOpenReaderImpl(pPrevReader);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
+ code = doOpenReaderImpl(pPrevReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
- code = doOpenReaderImpl(pNextReader);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
+ code = doOpenReaderImpl(pNextReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
- code = doOpenReaderImpl(pReader);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
+ code = doOpenReaderImpl(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ return code;
+ }
}
}
@@ -3513,6 +3514,10 @@ void tsdbReaderClose(STsdbReader* pReader) {
taosMemoryFree(pLReader);
}
+ if (pReader->innerReader[0] != 0) {
+ tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr);
+ }
+
tsdbDebug(
"%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64
" SMA-time:%.2f ms, fileBlocks:%" PRId64
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index 911bd6b02c..84b3afe72c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -124,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
}
// check
- if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
+ if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
}
@@ -843,7 +843,8 @@ _err:
// SDataFReader ====================================================
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
int32_t code = 0;
- SDataFReader *pReader;
+ int32_t lino = 0;
+ SDataFReader *pReader = NULL;
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
char fname[TSDB_FILENAME_LEN];
@@ -851,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->pTsdb = pTsdb;
pReader->pSet = pSet;
@@ -859,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// head
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// data
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// sma
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// stt
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- *ppReader = pReader;
- return code;
+_exit:
+ if (code) {
+ *ppReader = NULL;
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
-_err:
- tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
- *ppReader = NULL;
+ if (pReader) {
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]);
+ tsdbCloseFile(&pReader->pSmaFD);
+ tsdbCloseFile(&pReader->pDataFD);
+ tsdbCloseFile(&pReader->pHeadFD);
+ taosMemoryFree(pReader);
+ }
+ } else {
+ *ppReader = pReader;
+ }
return code;
}
@@ -1588,4 +1598,4 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
_err:
tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
return code;
-}
\ No newline at end of file
+}
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 423ea2576e..74c704598d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -445,13 +445,14 @@ _err:
int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) {
int32_t code = 0;
+ int32_t lino = 0;
STsdbSnapReader* pReader = NULL;
// alloc
pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->pTsdb = pTsdb;
pReader->sver = sver;
@@ -461,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
code = taosThreadRwlockRdlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFSRef(pTsdb, &pReader->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = taosThreadRwlockUnlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
// data
@@ -485,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pIter->aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = tBlockDataCreate(&pIter->bData);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pReader->bData);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// del
pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
if (pReader->aDelIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
if (pReader->aDelData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
- *ppReader = pReader;
- return code;
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code),
+ pTsdb->path);
+ *ppReader = NULL;
-_err:
- tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
- tstrerror(code));
- *ppReader = NULL;
+ if (pReader) {
+ taosArrayDestroy(pReader->aDelData);
+ taosArrayDestroy(pReader->aDelIdx);
+ tBlockDataDestroy(&pReader->bData, 1);
+ tsdbFSDestroy(&pReader->fs);
+ taosMemoryFree(pReader);
+ }
+ } else {
+ *ppReader = pReader;
+ tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
+ }
return code;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c
index 730bd264a7..682f2b4225 100644
--- a/source/dnode/vnode/src/vnd/vnodeBufPool.c
+++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c
@@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) {
void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
SVBufPoolNode *pNode;
void *p = NULL;
+ ASSERT(pPool != NULL);
+
taosThreadSpinLock(&pPool->lock);
if (pPool->node.size >= pPool->ptr - pPool->node.data + size) {
// allocate from the anchor node
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 07d9b96261..07c4c32955 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) {
int vnodeShouldCommit(SVnode *pVnode) {
if (pVnode->inUse) {
- return pVnode->inUse->size > pVnode->inUse->node.size;
+ return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size);
}
return false;
}
@@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
data = NULL;
if (vnodeEncodeInfo(pInfo, &data) < 0) {
+ vError("failed to encode json info.");
return -1;
}
@@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
}
if (taosWriteFile(pFile, data, strlen(data)) < 0) {
- vError("failed to write info file:%s data:%s", fname, terrstr());
+ vError("failed to write info file:%s error:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
@@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) {
snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path);
}
if (vnodeSaveInfo(dir, &info) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
walBeginSnapshot(pVnode->pWal, pVnode->state.applied);
// preCommit
// smaSyncPreCommit(pVnode->pSma);
- if (smaAsyncPreCommit(pVnode->pSma) < 0) {
- ASSERT(0);
+ if(smaAsyncPreCommit(pVnode->pSma) < 0){
+ vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
@@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) {
// commit each sub-system
if (metaCommit(pVnode->pMeta) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (VND_IS_RSMA(pVnode)) {
if (smaAsyncCommit(pVnode->pSma) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbCommit(VND_RSMA0(pVnode)) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbCommit(VND_RSMA1(pVnode)) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
if (tsdbCommit(VND_RSMA2(pVnode)) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
} else {
if (tsdbCommit(pVnode->pTsdb) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
}
if (tqCommit(pVnode->pTq) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
// walCommit (TODO)
// commit info
if (vnodeCommitInfo(dir, &info) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
@@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) {
// postCommit
// smaSyncPostCommit(pVnode->pSma);
if (smaAsyncPostCommit(pVnode->pSma) < 0) {
- ASSERT(0);
+ vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 298653d3ed..001bb5f7f2 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg));
if (pVnode->pWal == NULL) {
- vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno));
+ vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir);
goto _err;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index b83f790a3d..0f93e650c6 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -43,14 +43,23 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
tStartDecode(&dc);
- tDecodeI32v(&dc, &nReqs);
+ if (tDecodeI32v(&dc, &nReqs) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ goto _err;
+ }
for (int32_t iReq = 0; iReq < nReqs; iReq++) {
tb_uid_t uid = tGenIdPI64();
char *name = NULL;
tStartDecode(&dc);
- tDecodeI32v(&dc, NULL);
- tDecodeCStr(&dc, &name);
+ if (tDecodeI32v(&dc, NULL) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
+ if (tDecodeCStr(&dc, &name) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
*(int64_t *)(dc.data + dc.pos) = uid;
*(int64_t *)(dc.data + dc.pos + 8) = ctime;
@@ -81,10 +90,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
char *name = NULL;
tDecoderInit(&dc, pBlock->data, msgIter.schemaLen);
- tStartDecode(&dc);
+ if (tStartDecode(&dc) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
- tDecodeI32v(&dc, NULL);
- tDecodeCStr(&dc, &name);
+ if (tDecodeI32v(&dc, NULL) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
+ if (tDecodeCStr(&dc, &name) < 0) {
+ code = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
uid = metaGetTableEntryUidByName(pVnode->pMeta, name);
if (uid == 0) {
@@ -148,6 +166,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
int32_t len;
int32_t ret;
+ if (!pVnode->inUse) {
+ terrno = TSDB_CODE_VND_NOT_SYNCED;
+ vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr());
+ return -1;
+ }
+
vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
version);
@@ -268,10 +292,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
_do_commit:
vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
// commit current change
- vnodeCommit(pVnode);
+ if (vnodeCommit(pVnode) < 0) {
+ vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
+ }
// start a new one
- vnodeBegin(pVnode);
+ if (vnodeBegin(pVnode) < 0) {
+ vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
+ }
}
return 0;
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index d6e0de0eb1..fc996a6003 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -212,6 +212,7 @@ typedef struct SExprSupp {
int32_t numOfExprs; // the number of scalar expression in group operator
SqlFunctionCtx* pCtx;
int32_t* rowEntryInfoOffset; // offset value for each row result cell info
+ SFilterInfo* pFilterInfo;
} SExprSupp;
typedef struct SOperatorInfo {
@@ -926,7 +927,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
void doSetOperatorCompleted(SOperatorInfo* pOperator);
-void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo);
+void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
SSDataBlock* pBlock, const char* idStr);
diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c
index 6a43ea40c6..ce0ff485ab 100644
--- a/source/libs/executor/src/dataInserter.c
+++ b/source/libs/executor/src/dataInserter.c
@@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
}
pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows;
- qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows,
+ qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows,
pInserter->submitRes.affectedRows);
tFreeSSubmitRsp(pInserter->submitRes.pRsp);
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 0fadacb2c2..8277a4ed88 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1201,14 +1201,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) {
pExp->base.resSchema =
createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName);
- pExp->pExpr->_function.functionId = pFuncNode->funcId;
- pExp->pExpr->_function.pFunctNode = pFuncNode;
+ tExprNode* pExprNode = pExp->pExpr;
+
+ pExprNode->_function.functionId = pFuncNode->funcId;
+ pExprNode->_function.pFunctNode = pFuncNode;
+
+ tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName));
- strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
- tListLen(pExp->pExpr->_function.functionName));
#if 1
// todo refactor: add the parameter for tbname function
- if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
+ const char* name = "tbname";
+ int32_t len = strlen(name);
+
+ if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) &&
+ pExprNode->_function.functionName[len] == 0) {
+
pFuncNode->pParameterList = nodesMakeList();
ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index e527f7e7c1..413813f58a 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
pTaskInfo->streamInfo.schema = mtInfo.schema;
- qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts);
+ qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts);
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
SStreamRawScanInfo* pInfo = pOperator->info;
SSnapContext* sContext = pInfo->sContext;
if (setForSnapShot(sContext, pOffset->uid) != 0) {
- qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid);
+ qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
return -1;
}
- qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid);
+ qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts);
} else if (pOffset->type == TMQ_OFFSET__LOG) {
SStreamRawScanInfo* pInfo = pOperator->info;
tsdbReaderClose(pInfo->dataReader);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 8c1104f519..e59755787a 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1113,15 +1113,24 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep,
int32_t status);
-void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) {
+void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo) {
if (pFilterNode == NULL || pBlock->info.rows == 0) {
return;
}
- SFilterInfo* filter = NULL;
+ SFilterInfo* filter = pFilterInfo;
+ int64_t st = taosGetTimestampUs();
+
+// pError("start filter");
// todo move to the initialization function
- int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
+ int32_t code = 0;
+ bool needFree = false;
+ if (filter == NULL) {
+ needFree = true;
+ code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
+ }
+
SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(filter, ¶m1);
@@ -1130,7 +1139,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM
// todo the keep seems never to be True??
bool keep = filterExecute(filter, pBlock, &p, NULL, param1.numOfCols, &status);
- filterFreeInfo(filter);
+
+ if (needFree) {
+ filterFreeInfo(filter);
+ }
extractQualifiedTupleByFilterResult(pBlock, p, keep, status);
@@ -1845,6 +1857,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq));
if (NULL == pMsg) {
pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+ taosMemoryFree(pWrapper);
return pTaskInfo->code;
}
@@ -2479,7 +2492,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf);
- doFilter(pAggInfo->pCondition, pInfo->pRes, NULL);
+ doFilter(pAggInfo->pCondition, pInfo->pRes, NULL, NULL);
if (!hasRemainResults(&pAggInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
@@ -2873,7 +2886,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
break;
}
- doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo);
+ doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo, NULL);
if (fillResult->info.rows > 0) {
break;
}
@@ -3049,6 +3062,12 @@ void cleanupExprSupp(SExprSupp* pSupp) {
destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs);
taosMemoryFreeClear(pSupp->pExprInfo);
}
+
+ if (pSupp->pFilterInfo != NULL) {
+ filterFreeInfo(pSupp->pFilterInfo);
+ pSupp->pFilterInfo = NULL;
+ }
+
taosMemoryFree(pSupp->rowEntryInfoOffset);
}
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index b0bb9e42b9..e62f72c8f8 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -311,7 +311,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
SSDataBlock* pRes = pInfo->binfo.pRes;
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
- doFilter(pInfo->pCondition, pRes, NULL);
+ doFilter(pInfo->pCondition, pRes, NULL, NULL);
if (!hasRemainResults(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c
index b4aa6de0a8..53cfa6c27a 100644
--- a/source/libs/executor/src/joinoperator.c
+++ b/source/libs/executor/src/joinoperator.c
@@ -387,7 +387,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
break;
}
if (pJoinInfo->pCondAfterMerge != NULL) {
- doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL);
+ doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL, NULL);
}
if (pRes->info.rows >= pOperator->resultInfo.threshold) {
break;
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index e9e6fed66a..fe6ee4129a 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -315,7 +315,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
}
// do apply filter
- doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL);
+ doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL, NULL);
// when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint.
if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) {
@@ -325,7 +325,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
} else {
// do apply filter
if (pRes->info.rows > 0) {
- doFilter(pProjectInfo->pFilterNode, pRes, NULL);
+ doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL);
if (pRes->info.rows == 0) {
continue;
}
@@ -518,7 +518,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
}
}
- doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL);
+ doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL, NULL);
size_t rows = pInfo->pRes->info.rows;
if (rows > 0 || pOperator->status == OP_EXEC_DONE) {
break;
@@ -620,7 +620,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) {
}
pRes->info.rows = 1;
- doFilter(pProjectInfo->pFilterNode, pRes, NULL);
+ doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL);
/*int32_t status = */ doIngroupLimitOffset(&pProjectInfo->limitInfo, 0, pRes, pOperator);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 7ede06e6ca..66fad27d71 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -385,7 +385,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
if (pTableScanInfo->pFilterNode != NULL) {
int64_t st = taosGetTimestampUs();
- doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo);
+ doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, pOperator->exprSupp.pFilterInfo);
double el = (taosGetTimestampUs() - st) / 1000.0;
pTableScanInfo->readRecorder.filterTime += el;
@@ -754,6 +754,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
pInfo->pResBlock = createResDataBlock(pDescNode);
pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
+
+ if (pInfo->pFilterNode != NULL) {
+ code = filterInitFromNode((SNode*)pInfo->pFilterNode, &pOperator->exprSupp.pFilterInfo, 0);
+ }
+
pInfo->scanFlag = MAIN_SCAN;
pInfo->pColMatchInfo = pColList;
pInfo->currentGroupId = -1;
@@ -1123,7 +1128,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
return NULL;
}
- doFilter(pInfo->pCondition, pResult, NULL);
+ doFilter(pInfo->pCondition, pResult, NULL, NULL);
if (pResult->info.rows == 0) {
continue;
}
@@ -1474,7 +1479,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
}
if (filter) {
- doFilter(pInfo->pCondition, pInfo->pRes, NULL);
+ doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL);
}
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
blockDataFreeRes((SSDataBlock*)pBlock);
@@ -1896,7 +1901,7 @@ FETCH_NEXT_BLOCK:
}
}
- doFilter(pInfo->pCondition, pInfo->pRes, NULL);
+ doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL);
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) {
@@ -2382,7 +2387,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes;
}
- doFilter(pInfo->pCondition, pInfo->pRes, NULL);
+ doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL);
return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes;
}
@@ -2563,16 +2568,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
SMetaReader smrChildTable = {0};
metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
- metaGetTableEntryByName(&smrChildTable, condTableName);
+ int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
+ if (code != TSDB_CODE_SUCCESS) {
+ // terrno has been set by metaGetTableEntryByName, therefore, return directly
+ return NULL;
+ }
+
if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
metaReaderClear(&smrChildTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
+
SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK);
- metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+ code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+ if (code != TSDB_CODE_SUCCESS) {
+ // terrno has been set by metaGetTableEntryByUid
+ return NULL;
+ }
+
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smrSuperTable);
metaReaderClear(&smrChildTable);
@@ -3455,7 +3471,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
if (pTableScanInfo->pFilterNode != NULL) {
int64_t st = taosGetTimestampMs();
- doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo);
+ doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, NULL);
double el = (taosGetTimestampUs() - st) / 1000.0;
pTableScanInfo->readRecorder.filterTime += el;
@@ -3790,6 +3806,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
+ taosArrayDestroy(pColList);
goto _error;
}
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index ed3247e10e..0d5f521034 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
return NULL;
}
- doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo);
+ doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo, NULL);
if (blockDataGetNumOfRows(pBlock) == 0) {
continue;
}
@@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataDestroy(p);
- qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
+ qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
pDataBlock->info.rows);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}
diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index 5be9918185..8b716531e5 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -1501,7 +1501,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
}
doStreamFillImpl(pOperator);
- doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo);
+ doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo, NULL);
pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
if (pInfo->pRes->info.rows > 0) {
break;
@@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
&numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
pInfo->pCondition = pPhyFillNode->node.pConditions;
pInfo->pColMatchColInfo = pColMatchColInfo;
- initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+ int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
pInfo->srcRowIndex = 0;
pOperator->name = "FillOperator";
@@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo,
NULL, NULL, NULL);
- int32_t code = appendDownstream(pOperator, &downstream, 1);
+ code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
_error:
destroyStreamFillOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
+ pTaskInfo->code = code;
return NULL;
}
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index df16914cb8..9c219d2765 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1277,7 +1277,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
- doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
+ doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@@ -1315,7 +1315,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
- doFilter(pInfo->pCondition, pBlock, NULL);
+ doFilter(pInfo->pCondition, pBlock, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* pGpDatas = (uint64_t*)pGpCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
- SResultRowInfo dumyInfo;
+ SResultRowInfo dumyInfo = {0};
dumyInfo.cur.pageId = -1;
+
STimeWindow win = {0};
if (IS_FINAL_OP(pInfo)) {
win.skey = startTsCols[i];
@@ -2019,7 +2020,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
if (pOperator->status == OP_RES_TO_RETURN) {
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
- doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
+ doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@@ -2062,7 +2063,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
- doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
+ doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL);
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
if (!hasRemain) {
@@ -5251,7 +5252,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true);
doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes);
- doFilter(pMiaInfo->pCondition, pRes, NULL);
+ doFilter(pMiaInfo->pCondition, pRes, NULL, NULL);
if (pRes->info.rows >= pOperator->resultInfo.capacity) {
break;
}
@@ -5828,27 +5829,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
pInfo->isFinal = false;
- if (pIntervalPhyNode->window.pExprs != NULL) {
- int32_t numOfScalar = 0;
- SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
- int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
- }
+
pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
initResultSizeInfo(&pOperator->resultInfo, 4096);
SExprSupp* pSup = &pOperator->exprSupp;
+
+ initBasicInfo(&pInfo->binfo, pResBlock);
+ initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
+ initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
+
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
- initBasicInfo(&pInfo->binfo, pResBlock);
- initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
- initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
+ if (pIntervalPhyNode->window.pExprs != NULL) {
+ int32_t numOfScalar = 0;
+ SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
+ code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+ }
pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
pInfo->invertible = false;
diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c
index cdc4e66e42..9e89955ae5 100644
--- a/source/libs/nodes/src/nodesMsgFuncs.c
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@@ -478,7 +478,10 @@ static int32_t tlvDecodeValueEnum(STlvDecoder* pDecoder, void* pValue, int16_t l
return code;
}
-static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) {
+static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue, int32_t size) {
+ if (pTlv->len > size - 1) {
+ return TSDB_CODE_FAILED;
+ }
memcpy(pValue, pTlv->value, pTlv->len);
return TSDB_CODE_SUCCESS;
}
@@ -919,9 +922,14 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) {
}
break;
}
- case TSDB_DATA_TYPE_JSON:
+ case TSDB_DATA_TYPE_JSON: {
+ if (pTlv->len <= 0 || pTlv->len > TSDB_MAX_JSON_TAG_LEN) {
+ code = TSDB_CODE_FAILED;
+ break;
+ }
code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p);
break;
+ }
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
@@ -1097,7 +1105,7 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node);
break;
case FUNCTION_CODE_FUNCTION_NAME:
- code = tlvDecodeCStr(pTlv, pNode->functionName);
+ code = tlvDecodeCStr(pTlv, pNode->functionName, sizeof(pNode->functionName));
break;
case FUNCTION_CODE_FUNCTION_ID:
code = tlvDecodeI32(pTlv, &pNode->funcId);
@@ -1226,10 +1234,10 @@ static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeI32(pTlv, &pNode->acctId);
break;
case NAME_CODE_DB_NAME:
- code = tlvDecodeCStr(pTlv, pNode->dbname);
+ code = tlvDecodeCStr(pTlv, pNode->dbname, sizeof(pNode->dbname));
break;
case NAME_CODE_TABLE_NAME:
- code = tlvDecodeCStr(pTlv, pNode->tname);
+ code = tlvDecodeCStr(pTlv, pNode->tname, sizeof(pNode->tname));
break;
default:
break;
@@ -1538,7 +1546,7 @@ static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) {
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case EP_CODE_FQDN:
- code = tlvDecodeCStr(pTlv, pNode->fqdn);
+ code = tlvDecodeCStr(pTlv, pNode->fqdn, sizeof(pNode->fqdn));
break;
case EP_CODE_port:
code = tlvDecodeU16(pTlv, &pNode->port);
@@ -3207,7 +3215,7 @@ static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeI8(pTlv, &pNode->tableType);
break;
case PHY_QUERY_INSERT_CODE_TABLE_NAME:
- code = tlvDecodeCStr(pTlv, pNode->tableName);
+ code = tlvDecodeCStr(pTlv, pNode->tableName, sizeof(pNode->tableName));
break;
case PHY_QUERY_INSERT_CODE_VG_ID:
code = tlvDecodeI32(pTlv, &pNode->vgId);
@@ -3284,10 +3292,10 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) {
code = tlvDecodeI8(pTlv, &pNode->tableType);
break;
case PHY_DELETER_CODE_TABLE_FNAME:
- code = tlvDecodeCStr(pTlv, pNode->tableFName);
+ code = tlvDecodeCStr(pTlv, pNode->tableFName, sizeof(pNode->tableFName));
break;
case PHY_DELETER_CODE_TS_COL_NAME:
- code = tlvDecodeCStr(pTlv, pNode->tsColName);
+ code = tlvDecodeCStr(pTlv, pNode->tsColName, sizeof(pNode->tsColName));
break;
case PHY_DELETER_CODE_DELETE_TIME_RANGE:
code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange);
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 9196f77124..bc0c4d42bb 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -209,6 +209,10 @@ int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) {
}
SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId);
+ if (NULL == pAllocator) {
+ nodesError("allocator id %" PRIx64 " weak reference failed", allocatorId);
+ return -1;
+ }
return pAllocator->self;
}
@@ -1716,9 +1720,10 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = 0;
if ('\0' == pCol->tableAlias[0]) {
- len = sprintf(name, "%s", pCol->colName);
+ len = snprintf(name, sizeof(name), "%s", pCol->colName);
+ } else {
+ len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName);
}
- len = sprintf(name, "%s.%s", pCol->tableAlias, pCol->colName);
if (NULL == taosHashGet(pCxt->pColHash, name, len)) {
pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h
index 787abf287e..fb4b46aa35 100644
--- a/source/libs/parser/inc/parToken.h
+++ b/source/libs/parser/inc/parToken.h
@@ -175,8 +175,6 @@ _end:
void taosCleanupKeywordsTable();
-SToken tscReplaceStrToken(char **str, SToken *token, const char *newToken);
-
SToken taosTokenDup(SToken *pToken, char *buf, int32_t len);
#ifdef __cplusplus
diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 2e6f839368..15b939f14b 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -518,7 +518,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa
if (NULL != pDbName) {
COPY_STRING_FORM_ID_TOKEN(realTable->table.dbName, pDbName);
} else {
- strcpy(realTable->table.dbName, pCxt->pQueryCxt->db);
+ snprintf(realTable->table.dbName, sizeof(realTable->table.dbName), "%s", pCxt->pQueryCxt->db);
}
if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) {
COPY_STRING_FORM_ID_TOKEN(realTable->table.tableAlias, pTableAlias);
@@ -599,7 +599,7 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) {
state->pCol = createPrimaryKeyCol(pCxt, NULL);
if (NULL == state->pCol) {
nodesDestroyNode((SNode*)state);
- CHECK_OUT_OF_MEM(state->pCol);
+ CHECK_OUT_OF_MEM(NULL);
}
state->pExpr = pExpr;
return (SNode*)state;
@@ -613,7 +613,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode
interval->pCol = createPrimaryKeyCol(pCxt, NULL);
if (NULL == interval->pCol) {
nodesDestroyNode((SNode*)interval);
- CHECK_OUT_OF_MEM(interval->pCol);
+ CHECK_OUT_OF_MEM(NULL);
}
interval->pInterval = pInterval;
interval->pOffset = pOffset;
@@ -631,7 +631,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) {
fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == fill->pWStartTs) {
nodesDestroyNode((SNode*)fill);
- CHECK_OUT_OF_MEM(fill->pWStartTs);
+ CHECK_OUT_OF_MEM(NULL);
}
strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart");
return (SNode*)fill;
@@ -1495,10 +1495,10 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno
CHECK_OUT_OF_MEM(pStmt);
pStmt->indexType = type;
pStmt->ignoreExists = ignoreExists;
- strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName);
- strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName);
- strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName);
- strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName);
+ snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName);
+ snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName);
+ snprintf(pStmt->dbName, sizeof(pStmt->dbName), "%s", ((SRealTableNode*)pRealTable)->table.dbName);
+ snprintf(pStmt->tableName, sizeof(pStmt->tableName), "%s", ((SRealTableNode*)pRealTable)->table.tableName);
nodesDestroyNode(pIndexName);
nodesDestroyNode(pRealTable);
pStmt->pCols = pCols;
@@ -1524,8 +1524,8 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode*
SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->ignoreNotExists = ignoreNotExists;
- strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName);
- strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName);
+ snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName);
+ snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName);
nodesDestroyNode(pIndexName);
return (SNode*)pStmt;
}
@@ -1818,7 +1818,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb
SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
CHECK_OUT_OF_MEM(pFunc);
- strcpy(pFunc->functionName, pFuncName);
+ snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName);
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) {
nodesDestroyNode((SNode*)pFunc);
CHECK_OUT_OF_MEM(NULL);
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index eec20f91e4..5b861ef79a 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -378,7 +378,9 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time
} else if (pToken->type == TK_TODAY) {
ts = taosGetTimestampToday(timePrec);
} else if (pToken->type == TK_NK_INTEGER) {
- toInteger(pToken->z, pToken->n, 10, &ts);
+ if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) {
+ return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+ }
} else { // parse the RFC-3339/ISO-8601 timestamp format string
if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
@@ -591,8 +593,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
case TSDB_DATA_TYPE_BIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
- } else if (!IS_VALID_BIGINT(iv)) {
- return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z);
}
return func(pMsgBuf, &iv, pSchema->bytes, param);
}
@@ -894,10 +894,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16
case TSDB_DATA_TYPE_BIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
- } else if (!IS_VALID_BIGINT(iv)) {
- return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z);
}
-
val->i64 = iv;
break;
}
@@ -1173,7 +1170,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName*
NEXT_TOKEN(pCxt->pSql, sToken);
SName sname;
- createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
+ CHECK_CODE(createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg));
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&sname, dbFName);
strcpy(pCxt->sTableName, sname.tname);
@@ -1605,7 +1602,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
if (!pCxt->pComCxt->needMultiParse) {
continue;
} else {
- parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
+ parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId);
break;
}
}
@@ -1613,7 +1610,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
}
- parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
+ parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags));
@@ -1650,7 +1647,7 @@ static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) {
pCxt->pComCxt->needMultiParse = false;
return TSDB_CODE_SUCCESS;
}
- parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
+ parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum);
// merge according to vgId
if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks));
diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c
index e8c877bed2..997e680c99 100644
--- a/source/libs/parser/src/parInsertData.c
+++ b/source/libs/parser/src/parInsertData.c
@@ -12,7 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
-// clang-format off
+
#include "parInsertData.h"
#include "catalog.h"
@@ -25,8 +25,8 @@
(((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
typedef struct SBlockKeyTuple {
- TSKEY skey;
- void* payloadAddr;
+ TSKEY skey;
+ void* payloadAddr;
int16_t index;
} SBlockKeyTuple;
@@ -194,8 +194,8 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star
int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) {
SEncoder coder = {0};
- char* pBuf;
- int32_t len;
+ char* pBuf;
+ int32_t len;
int32_t ret = 0;
tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret);
@@ -211,19 +211,19 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq)
}
}
- pBuf= pBlocks->pData + pBlocks->size;
+ pBuf = pBlocks->pData + pBlocks->size;
tEncoderInit(&coder, pBuf, len);
- tEncodeSVCreateTbReq(&coder, pCreateTbReq);
+ int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq);
tEncoderClear(&coder);
-
pBlocks->size += len;
pBlocks->createTbReqLen = len;
- return TSDB_CODE_SUCCESS;
+
+ return code;
}
-int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize,
- STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList,
+int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset,
+ int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList,
SVCreateTbReq* pCreateTbReq) {
*dataBlocks = NULL;
STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen);
@@ -272,12 +272,12 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) {
}
taosMemoryFreeClear(pDataBlock->pData);
-// if (!pDataBlock->cloned) {
- // free the refcount for metermeta
- taosMemoryFreeClear(pDataBlock->pTableMeta);
+ // if (!pDataBlock->cloned) {
+ // free the refcount for metermeta
+ taosMemoryFreeClear(pDataBlock->pTableMeta);
- destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
-// }
+ destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
+ // }
taosMemoryFreeClear(pDataBlock);
}
@@ -687,16 +687,16 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
STableDataBlocks** p = taosHashIterate(pHashObj, NULL);
STableDataBlocks* pOneTableBlock = *p;
SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock
- SBlockRowMerger *pBlkRowMerger = NULL;
+ SBlockRowMerger* pBlkRowMerger = NULL;
while (pOneTableBlock) {
SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData;
if (pBlocks->numOfRows > 0) {
STableDataBlocks* dataBuf = NULL;
- pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId
- int32_t ret =
- getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0,
- pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL);
+ pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId
+ int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId),
+ TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf,
+ pVnodeDataBlockList, NULL);
if (ret != TSDB_CODE_SUCCESS) {
tdFreeSBlockRowMerger(pBlkRowMerger);
taosHashCleanup(pVnodeDataBlockHashList);
@@ -708,7 +708,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
// the maximum expanded size in byte when a row-wise data is converted to SDataRow format
int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize +
- sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen;
+ sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) +
+ pOneTableBlock->createTbReqLen;
if (dataBuf->nAllocSize < destSize) {
dataBuf->nAllocSize = (uint32_t)(destSize * 1.5);
@@ -861,7 +862,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) {
STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst);
if (pBlock->pTableMeta) {
- void *pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta));
+ void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta));
if (NULL == pNewMeta) {
taosMemoryFreeClear(*pDst);
return TSDB_CODE_OUT_OF_MEMORY;
@@ -887,20 +888,18 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI
}
pBlock->vgId = vgId;
-
+
if (pBlock->pTableMeta) {
pBlock->pTableMeta->uid = uid;
pBlock->pTableMeta->vgId = vgId;
}
-
+
memset(pBlock->pData, 0, sizeof(SSubmitBlk));
return TSDB_CODE_SUCCESS;
}
-STableMeta *qGetTableMetaInDataBlock(void* pDataBlock) {
- return ((STableDataBlocks*)pDataBlock)->pTableMeta;
-}
+STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; }
void qFreeStmtDataBlock(void* pDataBlock) {
if (pDataBlock == NULL) {
diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c
index 345b9cc5b9..76c01cfd90 100644
--- a/source/libs/parser/src/parTokenizer.c
+++ b/source/libs/parser/src/parTokenizer.c
@@ -613,27 +613,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
return 0;
}
-SToken tscReplaceStrToken(char** str, SToken* token, const char* newToken) {
- char* src = *str;
- size_t nsize = strlen(newToken);
- int32_t size = (int32_t)strlen(*str) - token->n + (int32_t)nsize + 1;
- int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src);
- SToken ntoken;
-
- *str = taosMemoryCalloc(1, size);
-
- strncpy(*str, src, bsize);
- strcat(*str, newToken);
- strcat(*str, token->z + token->n);
-
- ntoken.n = (uint32_t)nsize;
- ntoken.z = *str + bsize;
-
- taosMemoryFreeClear(src);
-
- return ntoken;
-}
-
SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) {
SToken t0 = {0};
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 2cef80c4cb..dbe0c97ab6 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -292,9 +292,13 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
} else {
do {
SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+ if (NULL == pTables) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
if (pCxt->currLevel == currTotalLevel) {
taosArrayPush(pTables, &pTable);
if (hasSameTableAlias(pTables)) {
+ taosArrayDestroy(pTables);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS,
"Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias);
}
@@ -308,7 +312,7 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) {
SFullDatabaseName name = {0};
- strcpy(name.fullDbName, pFullDbName);
+ snprintf(name.fullDbName, sizeof(name.fullDbName), "%s", pFullDbName);
return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName));
}
@@ -540,15 +544,17 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray
if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(pName, pCxt->pTables);
}
- if (pParCxt->async) {
- code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes);
- } else {
- SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter,
- .requestId = pParCxt->requestId,
- .requestObjRefId = pParCxt->requestRid,
- .mgmtEps = pParCxt->mgmtEpSet};
+ if (TSDB_CODE_SUCCESS == code) {
+ if (pParCxt->async) {
+ code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes);
+ } else {
+ SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter,
+ .requestId = pParCxt->requestId,
+ .requestObjRefId = pParCxt->requestRid,
+ .mgmtEps = pParCxt->mgmtEpSet};
- code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes);
+ code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes);
+ }
}
if (TSDB_CODE_SUCCESS != code) {
parserError("0x%" PRIx64 " getTableIndex error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId,
@@ -990,9 +996,9 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal)
return pCxt->errCode;
}
if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) {
- pVal->datum.i = pVal->datum.u;
+ pVal->datum.i = (int64_t)pVal->datum.u;
} else if (IS_FLOAT_TYPE(pVal->node.resType.type)) {
- pVal->datum.i = pVal->datum.d;
+ pVal->datum.i = (int64_t)pVal->datum.d;
} else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) {
pVal->datum.i = pVal->datum.b;
}
@@ -1069,7 +1075,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
}
case TSDB_DATA_TYPE_BIGINT: {
code = toInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.i);
- if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_BIGINT(pVal->datum.i))) {
+ if (strict && TSDB_CODE_SUCCESS != code) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
}
*(int64_t*)&pVal->typeData = pVal->datum.i;
@@ -3430,10 +3436,10 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) {
return NULL;
}
pCol->node.resType = ((SExprNode*)pNode)->resType;
- strcpy(pCol->tableAlias, pTableAlias);
- strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName);
- strcpy(pCol->node.aliasName, pCol->colName);
- strcpy(pCol->node.userAlias, ((SExprNode*)pNode)->userAlias);
+ snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pTableAlias);
+ snprintf(pCol->colName, sizeof(pCol->colName), "%s", ((SExprNode*)pNode)->aliasName);
+ snprintf(pCol->node.aliasName, sizeof(pCol->node.aliasName), "%s", pCol->colName);
+ snprintf(pCol->node.userAlias, sizeof(pCol->node.userAlias), "%s", ((SExprNode*)pNode)->userAlias);
return (SNode*)pCol;
}
@@ -3740,8 +3746,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int32_t val, int32_t minVal,
int32_t maxVal) {
if (val >= 0 && (val < minVal || val > maxVal)) {
- return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 " valid range: [%d, %d]", pName,
- val, minVal, maxVal);
+ return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %d valid range: [%d, %d]", pName, val,
+ minVal, maxVal);
}
return TSDB_CODE_SUCCESS;
}
@@ -3878,7 +3884,7 @@ static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pO
static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) {
if (val >= 0 && val != v1 && val != v2) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
- "Invalid option %s: %" PRId64 ", only %d, %d allowed", pName, val, v1, v2);
+ "Invalid option %s: %d, only %d, %d allowed", pName, val, v1, v2);
}
return TSDB_CODE_SUCCESS;
}
@@ -4463,8 +4469,8 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch
nodesDestroyNode((SNode*)pSelect);
return TSDB_CODE_OUT_OF_MEMORY;
}
- strcpy(pTable->table.dbName, pInfo->pDbName);
- strcpy(pTable->table.tableName, pInfo->pTableName);
+ snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName);
+ snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName);
TSWAP(pTable->pMeta, pInfo->pRollupTableMeta);
pSelect->pFromTable = (SNode*)pTable;
@@ -4940,9 +4946,9 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt
alterReq.superUser = 0;
alterReq.enable = pStmt->enable;
alterReq.sysInfo = pStmt->sysinfo;
- strcpy(alterReq.pass, pStmt->password);
+ snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password);
if (NULL != pCxt->pParseCxt->db) {
- strcpy(alterReq.dbname, pCxt->pParseCxt->db);
+ snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db);
}
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq);
@@ -6049,7 +6055,7 @@ static SNode* createProjectCol(const char* pProjCol) {
if (NULL == pCol) {
return NULL;
}
- strcpy(pCol->colName, pProjCol);
+ snprintf(pCol->colName, sizeof(pCol->colName), "%s", pProjCol);
return (SNode*)pCol;
}
@@ -6082,9 +6088,9 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32
nodesDestroyNode((SNode*)pSelect);
return TSDB_CODE_OUT_OF_MEMORY;
}
- strcpy(pRealTable->table.dbName, pDb);
- strcpy(pRealTable->table.tableName, pTable);
- strcpy(pRealTable->table.tableAlias, pTable);
+ snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb);
+ snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable);
+ snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable);
pSelect->pFromTable = (SNode*)pRealTable;
if (numOfProjs >= 0) {
@@ -7125,8 +7131,9 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p
pReq->newComment = strdup(pStmt->pOptions->comment);
if (NULL == pReq->newComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
+ } else {
+ pReq->newCommentLen = strlen(pReq->newComment);
}
- pReq->newCommentLen = strlen(pReq->newComment);
} else {
pReq->newCommentLen = -1;
}
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index dab1f01574..ee82e6c04b 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -381,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi
uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue,
strerror(errno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
+ taosMemoryFree(tmp);
goto end;
}
val.nData = valLen;
@@ -652,8 +653,8 @@ static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCac
}
SUserAuthInfo auth = {0};
- strcpy(auth.user, pCxt->pUser);
- strcpy(auth.dbFName, p->dbFName);
+ snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser);
+ snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName);
auth.type = AUTH_TYPE_WRITE;
taosArrayPush(pCatalogReq->pUser, &auth);
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index e149460550..9e1a01b13d 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -471,9 +471,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr)
return NULL;
}
pCol->node.resType = pExpr->resType;
- strcpy(pCol->colName, pExpr->aliasName);
+ snprintf(pCol->colName, sizeof(pCol->colName), "%s", pExpr->aliasName);
if (NULL != pStmtName) {
- strcpy(pCol->tableAlias, pStmtName);
+ snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName);
}
return pCol;
}
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 50f1f4b369..cd02e7a136 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -1124,7 +1124,7 @@ static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, SNodeList** pS
bool notOptimize = false;
int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, ¬Optimize, pSequencingNodes);
if (TSDB_CODE_SUCCESS != code || notOptimize) {
- nodesClearList(*pSequencingNodes);
+ NODES_CLEAR_LIST(*pSequencingNodes);
}
return code;
}
@@ -1361,74 +1361,6 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo
return code;
}
-static SNode* smaIndexOptCreateWStartTs() {
- SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
- if (NULL == pWStart) {
- return NULL;
- }
- strcpy(pWStart->functionName, "_wstart");
- snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart);
- if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pWStart, NULL, 0)) {
- nodesDestroyNode((SNode*)pWStart);
- return NULL;
- }
- return (SNode*)pWStart;
-}
-
-static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) {
- SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
- if (NULL == pMergeKey) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- pMergeKey->pExpr = nodesCloneNode(pCol);
- if (NULL == pMergeKey->pExpr) {
- nodesDestroyNode((SNode*)pMergeKey);
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- pMergeKey->order = ORDER_ASC;
- pMergeKey->nullOrder = NULL_ORDER_FIRST;
- return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey);
-}
-
-static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) {
- if (wstrartIndex < 0) {
- SNode* pWStart = smaIndexOptCreateWStartTs();
- if (NULL == pWStart) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- int32_t code = createColumnByRewriteExpr(pWStart, &pInterval->node.pTargets);
- if (TSDB_CODE_SUCCESS != code) {
- nodesDestroyNode(pWStart);
- return code;
- }
- wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1;
- }
- return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys);
-}
-
-static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
- SNodeList* pSmaCols, int32_t wstrartIndex) {
- SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent;
- SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets);
- if (NULL == pMergeTargets) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- SLogicNode* pSmaScan = NULL;
- SLogicNode* pMerge = NULL;
- SNodeList* pMergeKeys = NULL;
- int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys);
- if (TSDB_CODE_SUCCESS == code) {
- code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan);
- }
- if (TSDB_CODE_SUCCESS == code) {
- code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge);
- }
- if (TSDB_CODE_SUCCESS == code) {
- code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan);
- }
- return code;
-}
-
static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex,
SNodeList* pSmaCols, int32_t wstrartIndex) {
SLogicNode* pSmaScan = NULL;
@@ -1559,7 +1491,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) {
return NULL;
}
- strcpy(pFunc->functionName, pFuncName);
+ snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName);
if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) {
SColumnNode* pCol = (SColumnNode*)pNode;
partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName);
@@ -2028,7 +1960,7 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode
if (NULL != pSelectValue) {
strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName);
} else {
- snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc);
+ snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc);
}
int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol));
if (TSDB_CODE_SUCCESS == code) {
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index 81a3a2cb16..8ff8538c0d 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -67,7 +67,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const S
if (NULL == pSlot) {
return NULL;
}
- strcpy(pSlot->name, pName);
+ snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName);
pSlot->slotId = slotId;
pSlot->dataType = ((SExprNode*)pNode)->resType;
pSlot->reserve = reserve;
@@ -663,13 +663,17 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);
}
- SNodeList* condCols = nodesMakeList();
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
- code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols);
- }
- if (TSDB_CODE_SUCCESS == code) {
- code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc);
- nodesDestroyList(condCols);
+ SNodeList* pCondCols = nodesMakeList();
+ if (NULL == pCondCols) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ } else {
+ code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &pCondCols);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = addDataBlockSlots(pCxt, pCondCols, pJoin->node.pOutputDataBlockDesc);
+ }
+ nodesDestroyList(pCondCols);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
@@ -1634,7 +1638,7 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl
pSubplan->subplanType = pLogicSubplan->subplanType;
pSubplan->level = pLogicSubplan->level;
if (NULL != pCxt->pPlanCxt->pUser) {
- strcpy(pSubplan->user, pCxt->pPlanCxt->pUser);
+ snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser);
}
return pSubplan;
}
@@ -1825,7 +1829,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNode* pSubplan, int32_t lev
return TSDB_CODE_OUT_OF_MEMORY;
}
}
- return nodesListStrictAppend(pGroup->pNodeList, (SNode*)pSubplan);
+ return nodesListAppend(pGroup->pNodeList, (SNode*)pSubplan);
}
static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent,
diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index 74ed3b57a4..9567a2388f 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -342,7 +342,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pWStart->functionName, "_wstart");
- snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart);
+ snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart);
int32_t code = fmGetFuncInfo(pWStart, NULL, 0);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pFuncs, (SNode*)pWStart);
@@ -367,7 +367,7 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) {
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pWEnd->functionName, "_wend");
- snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, pWEnd);
+ snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd);
int32_t code = fmGetFuncInfo(pWEnd, NULL, 0);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd);
@@ -389,22 +389,19 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic
SNode* pConditions = pMergeWindow->node.pConditions;
pMergeWindow->node.pConditions = NULL;
- int32_t code = TSDB_CODE_SUCCESS;
SWindowLogicNode* pPartWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow);
if (NULL == pPartWin) {
- code = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
pPartWin->node.groupAction = GROUP_ACTION_KEEP;
+ pMergeWindow->node.pTargets = pTargets;
+ pMergeWindow->node.pConditions = pConditions;
+ pPartWin->node.pChildren = pChildren;
+ splSetParent((SLogicNode*)pPartWin);
- if (TSDB_CODE_SUCCESS == code) {
- pMergeWindow->node.pTargets = pTargets;
- pMergeWindow->node.pConditions = pConditions;
- pPartWin->node.pChildren = pChildren;
- splSetParent((SLogicNode*)pPartWin);
- code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs);
- }
int32_t index = 0;
+ int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplAppendWStart(pPartWin->pFuncs, &index);
}
@@ -721,15 +718,16 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO
SNode* pConditions = pMergeAgg->node.pConditions;
pMergeAgg->node.pConditions = NULL;
- int32_t code = TSDB_CODE_SUCCESS;
SAggLogicNode* pPartAgg = (SAggLogicNode*)nodesCloneNode((SNode*)pMergeAgg);
if (NULL == pPartAgg) {
- code = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_OUT_OF_MEMORY;
}
pPartAgg->node.groupAction = GROUP_ACTION_KEEP;
- if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) {
+ int32_t code = TSDB_CODE_SUCCESS;
+
+ if (NULL != pGroupKeys) {
pPartAgg->pGroupKeys = pGroupKeys;
code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets);
}
diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h
index 626e78b4d3..36c6817595 100644
--- a/source/libs/qworker/inc/qwInt.h
+++ b/source/libs/qworker/inc/qwInt.h
@@ -123,6 +123,7 @@ typedef struct SQWTaskCtx {
int32_t execId;
int32_t level;
+ bool queryGotData;
bool queryRsped;
bool queryEnd;
bool queryContinue;
diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c
index 9115c2d3aa..d2c16a1cb5 100644
--- a/source/libs/qworker/src/qworker.c
+++ b/source/libs/qworker/src/qworker.c
@@ -10,6 +10,7 @@
#include "tdatablock.h"
#include "tmsg.h"
#include "tname.h"
+#include "tglobal.h"
SQWorkerMgmt gQwMgmt = {
.lock = 0,
@@ -92,6 +93,19 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
return TSDB_CODE_SUCCESS;
}
+int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) {
+ if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) {
+ if (!ctx->localExec) {
+ qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx);
+ QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode));
+ }
+
+ ctx->queryRsped = true;
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
int32_t code = 0;
bool qcontinue = true;
@@ -144,7 +158,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
if (numOfResBlock == 0) {
QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds);
} else {
- QW_TASK_DLOG("qExecTask done", "");
+ QW_TASK_DLOG("qExecTask done, useconds:%" PRIu64, useconds);
}
dsEndPut(sinkHandle, useconds);
@@ -234,6 +248,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
int32_t code = 0;
SOutputData output = {0};
+ if (NULL == ctx->sinkHandle) {
+ return TSDB_CODE_SUCCESS;
+ }
+
*dataLen = 0;
while (true) {
@@ -407,6 +425,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION);
}
+ if (ctx->rspCode) {
+ QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase));
+ QW_ERR_JRET(ctx->rspCode);
+ }
+
if (!ctx->queryRsped) {
QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR);
@@ -419,6 +442,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
}
+ if (ctx->rspCode) {
+ QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase));
+ QW_ERR_JRET(ctx->rspCode);
+ }
+
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
@@ -499,21 +527,17 @@ _return:
if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC);
+ ctx->queryGotData = true;
}
- if (QW_PHASE_POST_QUERY == phase && ctx) {
- if (!ctx->localExec) {
- bool rsped = false;
- SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo};
- qwDbgSimulateRedirect(&qwMsg, ctx, &rsped);
- qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped);
- if (!rsped) {
- qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx);
- QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code));
- }
+ if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) {
+ bool rsped = false;
+ SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo};
+ qwDbgSimulateRedirect(&qwMsg, ctx, &rsped);
+ qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped);
+ if (!rsped) {
+ qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false);
}
-
- ctx->queryRsped = true;
}
if (ctx) {
@@ -551,6 +575,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx));
ctx->ctrlConnInfo = qwMsg->connInfo;
+ ctx->phase = -1;
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT));
@@ -604,6 +629,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
+ qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true);
+
ctx->level = plan->level;
atomic_store_ptr(&ctx->taskHandle, pTaskInfo);
atomic_store_ptr(&ctx->sinkHandle, sinkHandle);
@@ -619,6 +646,31 @@ _return:
input.msgType = qwMsg->msgType;
code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
+ if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
+ void *rsp = NULL;
+ int32_t dataLen = 0;
+ SOutputData sOutput = {0};
+ QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput));
+
+ if (rsp) {
+ bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
+
+ qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete);
+ if (qComplete) {
+ atomic_store_8((int8_t *)&ctx->queryEnd, true);
+ }
+
+ qwMsg->connInfo = ctx->dataConnInfo;
+ QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
+
+ qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code);
+ rsp = NULL;
+
+ QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code,
+ tstrerror(code), dataLen);
+ }
+ }
+
QW_RET(TSDB_CODE_SUCCESS);
}
@@ -740,7 +792,9 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
locked = true;
// RC WARNING
- if (QW_QUERY_RUNNING(ctx)) {
+ if (-1 == ctx->phase || false == ctx->queryGotData) {
+ QW_TASK_DLOG_E("task query unfinished");
+ } else if (QW_QUERY_RUNNING(ctx)) {
atomic_store_8((int8_t *)&ctx->queryContinue, 1);
} else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC);
diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c
index 2678a652a9..33bf50d853 100644
--- a/source/libs/scheduler/src/schTask.c
+++ b/source/libs/scheduler/src/schTask.c
@@ -913,7 +913,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask));
if (SCH_IS_QUERY_JOB(pJob)) {
- SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
+// SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
}
SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType));
@@ -993,6 +993,12 @@ int32_t schLaunchTaskImpl(void *param) {
SCH_ERR_JRET(schLaunchRemoteTask(pJob, pTask));
}
+#if 0
+ if (SCH_IS_QUERY_JOB(pJob)) {
+ SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask));
+ }
+#endif
+
_return:
if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) {
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 813a11c056..c4b45649a4 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -155,8 +155,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) {
return;
}
- SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", errCode:0x%x", *jobId, errCode);
-
+ SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode));
schHandleJobDrop(pJob, errCode);
schReleaseJob(*jobId);
diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c
index 7c4aaba6ce..5aadb599a3 100644
--- a/source/libs/stream/src/streamUpdate.c
+++ b/source/libs/stream/src/streamUpdate.c
@@ -174,14 +174,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
if (res == TSDB_CODE_SUCCESS) {
return false;
} else {
- qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64
- ", ts:%" PRIu64,
- tableId, maxTs, *pMapMaxTs, ts);
return true;
}
}
- qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64,
- tableId, maxTs, *pMapMaxTs, ts);
return true;
}
@@ -203,14 +198,10 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
}
if (ts < pInfo->minTS) {
- qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64,
- tableId, maxTs, *pMapMaxTs, ts);
return true;
} else if (res == TSDB_CODE_SUCCESS) {
return false;
}
- qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
- maxTs, *pMapMaxTs, ts);
// check from tsdb api
return true;
}
diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 0afc373f2d..101e99afea 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -169,7 +169,7 @@ typedef struct SSyncNode {
} SSyncNode;
// open/close --------------
-SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
+SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo);
void syncNodeStart(SSyncNode* pSyncNode);
void syncNodeStartStandBy(SSyncNode* pSyncNode);
void syncNodeClose(SSyncNode* pSyncNode);
diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c
index e000ba8bf8..4a35a15d3e 100644
--- a/source/libs/sync/src/syncAppendEntries.c
+++ b/source/libs/sync/src/syncAppendEntries.c
@@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset);
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset);
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c
index b604d25816..511113352e 100644
--- a/source/libs/sync/src/syncCommit.c
+++ b/source/libs/sync/src/syncCommit.c
@@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
} else {
pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index);
- ASSERT(pEntry != NULL);
+ if (pEntry == NULL) {
+ sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index);
+ return;
+ }
}
// cannot commit, even if quorum agree. need check term!
if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) {
@@ -127,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
// execute fsm
if (pSyncNode->pFsm != NULL) {
int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state);
- ASSERT(code == 0);
+ if (code != 0) {
+ wError("failed to commit sync node since %s", tstrerror(terrno));
+ }
}
}
}
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 378ac743b3..907eb21f4c 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -86,10 +86,10 @@ void syncCleanUp() {
}
}
-int64_t syncOpen(const SSyncInfo* pSyncInfo) {
+int64_t syncOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
if (pSyncNode == NULL) {
- sError("failed to open sync node. vgId:%d", pSyncInfo->vgId);
+ sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr());
return -1;
}
@@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg
return ret;
}
-int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
+int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) {
SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
if (pSyncNode == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
@@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
}
ASSERT(rid == pSyncNode->rid);
+#if 0
if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR;
@@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) {
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
return ret;
+#else
+ syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg);
+ syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID);
+ taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+ return 0;
+#endif
}
int32_t syncLeaderTransfer(int64_t rid) {
@@ -919,9 +926,7 @@ _END:
}
// open/close --------------
-SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
- SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo;
-
+SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode));
if (pSyncNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
sError("failed to open raft cfg file. path:%s", pSyncNode->configPath);
goto _error;
}
- pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
+ if (pSyncInfo->syncCfg.replicaNum == 0) {
+ pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
+ }
raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
@@ -2650,7 +2657,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
if (ths->state == TAOS_SYNC_STATE_LEADER) {
int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
- ASSERT(code == 0);
+ if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
+ return -1;
+ }
syncNodeReplicate(ths, false);
}
@@ -2726,7 +2736,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
if (code != 0) {
// del resp mgr, call FpCommitCb
- ASSERT(0);
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
return -1;
}
@@ -2790,8 +2800,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
if (code != 0) {
+ sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno));
// del resp mgr, call FpCommitCb
- ASSERT(0);
return -1;
}
@@ -3043,7 +3053,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex,
pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
} else {
code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry);
- ASSERT(code == 0);
+ if (code != 0) {
+ sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i);
+ return -1;
+ }
ASSERT(pEntry != NULL);
}
diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c
index 496c8419de..c3dad104d1 100644
--- a/source/libs/sync/src/syncRaftLog.c
+++ b/source/libs/sync/src/syncRaftLog.c
@@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pEntry->index, err, err, errStr, sysErr, sysErrStr);
syncNodeErrorLog(pData->pSyncNode, logBuf);
-
- ASSERT(0);
return -1;
}
pEntry->index = index;
@@ -414,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
return 0;
}
-SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
+SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
- taosThreadMutexLock(&(pData->mutex));
-
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReader* pWalHandle = pData->pWalHandle;
ASSERT(pWalHandle != NULL);
@@ -444,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
}
} while (0);
- ASSERT(0);
+ sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index);
+ return NULL;
}
SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
@@ -465,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
terrno = saveErr;
*/
- taosThreadMutexUnlock(&(pData->mutex));
return pEntry;
} else {
@@ -473,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
}
}
+SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
+ SSyncLogStoreData* pData = pLogStore->data;
+ SSyncRaftEntry *pEntry = NULL;
+
+ taosThreadMutexLock(&pData->mutex);
+ pEntry = logStoreGetEntryWithoutLock(pLogStore, index);
+ taosThreadMutexUnlock(&pData->mutex);
+ return pEntry;
+}
+
int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index a148c1e36b..13b7e0ac0a 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
ret = tdbPagerWrite(pPager, pPage);
if (ret < 0) {
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN
ret = tdbPagerWrite(pPager, pChild);
if (ret < 0) {
- // TODO
- ASSERT(0);
- return 0;
+ tdbError("failed to write page since %s", terrstr());
+ return -1;
}
// Copy the root page content to the child page
@@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
ret = tdbPagerWrite(pBt->pPager, pOlds[i]);
if (ret < 0) {
- // TODO
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
}
@@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
ret = tdbPagerWrite(pBt->pPager, pParent);
if (ret < 0) {
- // TODO
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
ret = tdbPagerWrite(pBt->pPager, pNews[iNew]);
if (ret < 0) {
- // TODO
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
}
@@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt)
// mark dirty
ret = tdbPagerWrite(pBt->pPager, *ppOfp);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -987,7 +984,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
// fetch a new ofp and make it dirty
SPgno pgno = 0;
- SPage *ofp, *nextOfp;
+ SPage *ofp = NULL, *nextOfp = NULL;
ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt);
if (ret < 0) {
@@ -1942,7 +1939,7 @@ int tdbBtcDelete(SBTC *pBtc) {
// drop the cell on the leaf
ret = tdbPagerWrite(pPager, pBtc->pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -1964,7 +1961,7 @@ int tdbBtcDelete(SBTC *pBtc) {
if (idx < nCells) {
ret = tdbPagerWrite(pPager, pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -2029,7 +2026,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int
// mark dirty
ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index ea16e80562..6c01348bc2 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) {
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerBegin(pPager, pTxn);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
return -1;
}
}
@@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) {
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerCommit(pPager, pTxn);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
return -1;
}
}
@@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) {
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerAbort(pPager, pTxn);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId);
return -1;
}
}
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 58be8642d7..dd2416e5b8 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -156,6 +156,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) {
ret = tdbPagerWrite(pPager, pPage);
if (ret < 0) {
+ tdbError("failed to write page since %s", terrstr());
return -1;
}
@@ -210,7 +211,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
ret = tdbPagerWritePageToJournal(pPager, pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page to journal since %s", tstrerror(terrno));
return -1;
}
}
@@ -226,6 +227,8 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
// Open the journal
pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (pPager->jfd < 0) {
+ tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -243,9 +246,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
// sync the journal file
ret = tdbOsFSync(pPager->jfd);
if (ret < 0) {
- // TODO
- ASSERT(0);
- return 0;
+ tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
}
// loop to write the dirty pages to file
@@ -255,7 +258,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
pPage = (SPage *)pNode;
ret = tdbPagerWritePageToDB(pPager, pPage);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1;
}
}
@@ -277,11 +280,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// sync the db file
- tdbOsFSync(pPager->fd);
+ if (tdbOsFSync(pPager->fd) < 0) {
+ tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
// remove the journal file
- tdbOsClose(pPager->jfd);
- tdbOsRemove(pPager->jFileName);
+ if (tdbOsClose(pPager->jfd) < 0) {
+ tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+ tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
pPager->inTran = 0;
return 0;
@@ -297,14 +314,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
// 0, sync the journal file
ret = tdbOsFSync(pPager->jfd);
if (ret < 0) {
- // TODO
- ASSERT(0);
- return 0;
+ tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
}
tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755);
if (jfd == NULL) {
- return 0;
+ return -1;
}
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
@@ -516,11 +533,15 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno));
if (ret < 0) {
+ tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize);
if (ret < 0) {
+ tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, pPage->pageSize);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -540,13 +561,15 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
- ASSERT(0);
+ tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, pPage->pageSize);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -590,23 +613,38 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
- ASSERT(0);
+ tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize);
if (ret < 0) {
- ASSERT(0);
+ tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize);
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
- tdbOsFSync(pPager->fd);
+ if (tdbOsFSync(pPager->fd) < 0) {
+ tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
tdbOsFree(pageBuf);
- tdbOsClose(jfd);
- tdbOsRemove(pPager->jFileName);
+ if (tdbOsClose(jfd) < 0) {
+ tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+ tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
return 0;
}
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 83323725be..359b9824cf 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -597,6 +597,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) {
static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SCliConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
+ tDebug("%s conn %p alloc read buf", CONN_GET_INST_LABEL(conn), conn);
transAllocBuffer(pBuf, buf);
}
static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
@@ -609,7 +610,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
if (nread > 0) {
pBuf->len += nread;
while (transReadComplete(pBuf)) {
- tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn);
+ tDebug("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn);
if (pBuf->invalid) {
cliHandleExcept(conn);
break;
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index 42079ff005..f843c8559c 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -203,7 +203,7 @@ bool transReadComplete(SConnBuffer* connBuf) {
}
int transSetConnOption(uv_tcp_t* stream) {
- uv_tcp_nodelay(stream, 1);
+ uv_tcp_nodelay(stream, 0);
int ret = uv_tcp_keepalive(stream, 5, 60);
return ret;
}
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 57bf8a34b3..3616b4808f 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -332,7 +332,10 @@ void uvOnSendCb(uv_write_t* req, int status) {
if (status == 0) {
tTrace("conn %p data already was written on stream", conn);
if (!transQueueEmpty(&conn->srvMsgs)) {
- SSvrMsg* msg = transQueuePop(&conn->srvMsgs);
+ SSvrMsg* msg = transQueuePop(&conn->srvMsgs);
+ STraceId* trace = &msg->msg.info.traceId;
+ tGDebug("conn %p write data out", conn);
+
destroySmsg(msg);
// send cached data
if (!transQueueEmpty(&conn->srvMsgs)) {
diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index 144cce2cd0..279f4dc656 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -35,113 +35,251 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe
int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; }
-static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
- sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
+static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
+ return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
}
-static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
- int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
- ASSERT(sz > 0);
-#if 0
- for (int i = 0; i < sz; i++) {
- SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
- }
-#endif
+static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) {
+ return sprintf(buf, "%s/meta-ver.tmp", pWal->path);
+}
- SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1);
+static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
+ int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+ terrno = TSDB_CODE_SUCCESS;
+ ASSERT(fileIdx >= 0 && fileIdx < sz);
+
+ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr);
+ walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL);
- int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize);
- pLastFileInfo->fileSize = fileSize;
- TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
+ TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
+ wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
+ // ensure size as non-negative
+ pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize);
+
uint64_t magic = WAL_MAGIC;
+ int64_t walCkHeadSz = sizeof(SWalCkHead);
+ int64_t end = fileSize;
+ int64_t offset = 0;
+ int32_t capacity = 0;
+ int32_t readSize = 0;
+ char* buf = NULL;
+ char* found = NULL;
+ bool firstTrial = pFileInfo->fileSize < fileSize;
- char* buf = taosMemoryMalloc(readSize + 5);
- if (buf == NULL) {
- taosCloseFile(&pFile);
- terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
- return -1;
- }
-
- int64_t offset;
- offset = taosLSeekFile(pFile, -readSize, SEEK_END);
- if (readSize != taosReadFile(pFile, buf, readSize)) {
- taosMemoryFree(buf);
- taosCloseFile(&pFile);
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
-
- char* found = NULL;
+ // search for the valid last WAL entry, e.g. block by block
while (1) {
+ offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE);
+ ASSERT(offset <= end);
+ readSize = end - offset;
+ capacity = readSize + sizeof(magic);
+
+ int64_t limit = WAL_RECOV_SIZE_LIMIT;
+ if (limit < readSize) {
+ wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64
+ ", end:%" PRId64 ", file:%s",
+ pWal->cfg.vgId, limit, offset, end, fnameStr);
+ terrno = TSDB_CODE_WAL_SIZE_LIMIT;
+ goto _err;
+ }
+
+ void* ptr = taosMemoryRealloc(buf, capacity);
+ if (ptr == NULL) {
+ terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
+ goto _err;
+ }
+ buf = ptr;
+
+ int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET);
+ if (ret < 0) {
+ wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (readSize != taosReadFile(pFile, buf, readSize)) {
+ wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ readSize, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ char* candidate = NULL;
char* haystack = buf;
- char* candidate;
- while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) {
- // read and validate
- SWalCkHead* logContent = (SWalCkHead*)candidate;
- if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) {
- found = candidate;
+
+ while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) {
+ // validate head
+ int64_t len = readSize - (candidate - buf);
+ if (len < walCkHeadSz) {
+ break;
}
+ SWalCkHead* logContent = (SWalCkHead*)candidate;
+ if (walValidHeadCksum(logContent) != 0) {
+ wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s",
+ ((char*)(logContent)-buf), fnameStr);
+ haystack = candidate + 1;
+ if (firstTrial) {
+ break;
+ } else {
+ continue;
+ }
+ }
+
+ // validate body
+ int64_t size = walCkHeadSz + logContent->head.bodyLen;
+ if (len < size) {
+ int64_t extraSize = size - len;
+ if (capacity < readSize + extraSize + sizeof(magic)) {
+ capacity += extraSize;
+ void* ptr = taosMemoryRealloc(buf, capacity);
+ if (ptr == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ buf = ptr;
+ }
+ int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET);
+ if (ret < 0) {
+ wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno),
+ offset);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+ if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) {
+ wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s",
+ pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+ }
+ if (walValidBodyCksum(logContent) != 0) {
+ terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
+ wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s",
+ ((char*)(logContent)-buf), fnameStr);
+ haystack = candidate + 1;
+ if (firstTrial) {
+ break;
+ } else {
+ continue;
+ }
+ }
+
+ // found one
+ found = candidate;
haystack = candidate + 1;
}
+
if (found || offset == 0) break;
- offset = TMIN(0, offset - readSize + sizeof(uint64_t));
- int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET);
- ASSERT(offset == offset2);
- if (readSize != taosReadFile(pFile, buf, readSize)) {
- taosMemoryFree(buf);
- taosCloseFile(&pFile);
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
-#if 0
- if (found == buf) {
- SWalCkHead* logContent = (SWalCkHead*)found;
- if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) {
- // file has to be deleted
- taosMemoryFree(buf);
- taosCloseFile(&pFile);
- terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
- return -1;
- }
- }
-#endif
+
+ // go backwards, e.g. by at most one WAL scan buf size
+ end = offset + walCkHeadSz - 1;
+ firstTrial = false;
}
- if (found == NULL) {
- // file corrupted, no complete log
- // TODO delete and search in previous files
- /*ASSERT(0);*/
- terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
- return -1;
+ // determine end of last entry
+ SWalCkHead* lastEntry = (SWalCkHead*)found;
+ int64_t retVer = -1;
+ int64_t lastEntryBeginOffset = 0;
+ int64_t lastEntryEndOffset = 0;
+
+ if (lastEntry == NULL) {
+ terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+ } else {
+ retVer = lastEntry->head.version;
+ lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf);
+ lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen;
}
// truncate file
- SWalCkHead* lastEntry = (SWalCkHead*)found;
- int64_t retVer = lastEntry->head.version;
- int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf);
- int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen;
if (lastEntryEndOffset != fileSize) {
- wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
+ wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
fileSize);
- taosFtruncateFile(pFile, lastEntryEndOffset);
- ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset;
- pWal->totSize -= (fileSize - lastEntryEndOffset);
+ if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) {
+ wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ if (taosFsyncFile(pFile) < 0) {
+ wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
}
+ pFileInfo->fileSize = lastEntryEndOffset;
taosCloseFile(&pFile);
taosMemoryFree(buf);
-
return retVer;
+
+_err:
+ taosCloseFile(&pFile);
+ taosMemoryFree(buf);
+ return -1;
+}
+
+static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
+ int metaFileNum = taosArrayGetSize(metaLogList);
+ int actualFileNum = taosArrayGetSize(actualLogList);
+ int j = 0;
+
+ // both of the lists in asc order
+ for (int i = 0; i < actualFileNum; i++) {
+ SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i);
+ while (j < metaFileNum) {
+ SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j);
+ ASSERT(pMetaInfo != NULL);
+ if (pMetaInfo->firstVer < pLogInfo->firstVer) {
+ j++;
+ } else if (pMetaInfo->firstVer == pLogInfo->firstVer) {
+ (*pLogInfo) = *pMetaInfo;
+ j++;
+ break;
+ } else {
+ break;
+ }
+ }
+ }
+
+ taosArrayClear(metaLogList);
+
+ for (int i = 0; i < actualFileNum; i++) {
+ SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i);
+ taosArrayPush(metaLogList, pFileInfo);
+ }
+}
+
+void walAlignVersions(SWal* pWal) {
+ if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) {
+ wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId,
+ pWal->vers.firstVer, pWal->vers.snapshotVer);
+ pWal->vers.firstVer = pWal->vers.snapshotVer + 1;
+ }
+ if (pWal->vers.lastVer < pWal->vers.snapshotVer) {
+ wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+ pWal->vers.lastVer, pWal->vers.snapshotVer);
+ pWal->vers.lastVer = pWal->vers.snapshotVer;
+ }
+ if (pWal->vers.commitVer < pWal->vers.snapshotVer) {
+ wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+ pWal->vers.commitVer, pWal->vers.snapshotVer);
+ pWal->vers.commitVer = pWal->vers.snapshotVer;
+ }
+ if (pWal->vers.appliedVer < pWal->vers.snapshotVer) {
+ wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId,
+ pWal->vers.appliedVer, pWal->vers.snapshotVer);
+ pWal->vers.appliedVer = pWal->vers.snapshotVer;
+ }
+
+ pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer);
+ pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer);
}
int walCheckAndRepairMeta(SWal* pWal) {
@@ -150,7 +288,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern;
regex_t idxRegPattern;
- bool fixed = false;
regcomp(&logRegPattern, logPattern, REG_EXTENDED);
regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
@@ -184,225 +321,237 @@ int walCheckAndRepairMeta(SWal* pWal) {
taosArraySort(actualLog, compareWalFileInfo);
- int metaFileNum = taosArrayGetSize(pWal->fileInfoSet);
- int actualFileNum = taosArrayGetSize(actualLog);
+ int metaFileNum = taosArrayGetSize(pWal->fileInfoSet);
+ int actualFileNum = taosArrayGetSize(actualLog);
+ int64_t firstVerPrev = pWal->vers.firstVer;
+ int64_t lastVerPrev = pWal->vers.lastVer;
+ int64_t totSize = 0;
+ bool updateMeta = (metaFileNum != actualFileNum);
-#if 0
- for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) {
- SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo);
+ // rebuild meta of file info
+ walRebuildFileInfoSet(pWal->fileInfoSet, actualLog);
+ taosArrayDestroy(actualLog);
+
+ int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+ ASSERT(sz == actualFileNum);
+
+ // scan and determine the lastVer
+ int32_t fileIdx = sz;
+
+ while (--fileIdx >= 0) {
char fnameStr[WAL_FILE_LEN];
+ int64_t fileSize = 0;
+ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
+
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
- int64_t fileSize = 0;
- taosStatFile(fnameStr, &fileSize, NULL);
- if (fileSize == 0) {
+ int32_t code = taosStatFile(fnameStr, &fileSize, NULL);
+ if (code < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
+ return -1;
+ }
+
+ ASSERT(pFileInfo->firstVer >= 0);
+
+ if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) {
+ totSize += pFileInfo->fileSize;
+ continue;
+ }
+ updateMeta = true;
+
+ int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx);
+ if (lastVer < 0) {
+ if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) {
+ wError("failed to scan wal last ver since %s", terrstr());
+ return -1;
+ }
+ ASSERT(pFileInfo->fileSize == 0);
+ // remove the empty wal log, and its idx
taosRemoveFile(fnameStr);
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
taosRemoveFile(fnameStr);
- taosArrayPop(pLogInfoArray);
- } else {
- break;
+ // remove its meta entry
+ taosArrayRemove(pWal->fileInfoSet, fileIdx);
+ continue;
}
+
+ // update lastVer
+ pFileInfo->lastVer = lastVer;
+ totSize += pFileInfo->fileSize;
}
- actualFileNum = taosArrayGetSize(pLogInfoArray);
-#endif
-
- {
- int32_t i = 0, j = 0;
- while (i < actualFileNum && j < metaFileNum) {
- SWalFileInfo* pActualFile = taosArrayGet(actualLog, i);
- SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j);
- if (pActualFile->firstVer < pMetaFile->firstVer) {
- char fNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- i++;
- } else if (pActualFile->firstVer > pMetaFile->firstVer) {
- taosArrayRemove(pWal->fileInfoSet, j);
- metaFileNum--;
- } else {
- i++;
- j++;
- }
- }
- if (i == actualFileNum && j == metaFileNum) {
- if (j > 0) {
- SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1);
- int64_t fsize = 0;
- char fNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pLastInfo->firstVer, fNameStr);
- taosStatFile(fNameStr, &fsize, NULL);
- if (pLastInfo->fileSize != fsize) {
- fixed = true;
- pLastInfo->fileSize = fsize;
- pLastInfo->lastVer = walScanLogGetLastVer(pWal);
- }
- }
- } else {
- fixed = true;
- while (i < actualFileNum) {
- SWalFileInfo* pActualFile = taosArrayGet(actualLog, i);
- char fNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pActualFile->firstVer, fNameStr);
- taosStatFile(fNameStr, &pActualFile->fileSize, NULL);
-
- if (pActualFile->fileSize == 0) {
- ASSERT(i == actualFileNum - 1);
- taosRemoveFile(fNameStr);
-
- walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- break;
- }
-
- if (i < actualFileNum - 1) {
- pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1;
- taosArrayPush(pWal->fileInfoSet, pActualFile);
- i++;
- } else {
- pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile);
- pActualFile->lastVer = walScanLogGetLastVer(pWal);
- if (pActualFile->lastVer == -1) {
- taosRemoveFile(fNameStr);
-
- walBuildIdxName(pWal, pActualFile->firstVer, fNameStr);
- taosRemoveFile(fNameStr);
- taosArrayPop(pWal->fileInfoSet);
- }
- break;
- }
- }
- }
- }
-
-#if 0
- if (metaFileNum > actualFileNum) {
- taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum);
- } else if (metaFileNum < actualFileNum) {
- for (int i = metaFileNum; i < actualFileNum; i++) {
- SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i);
- taosArrayPush(pWal->fileInfoSet, pFileInfo);
- }
- }
-#endif
-
- taosArrayDestroy(actualLog);
-
+ // reset vers info and so on
actualFileNum = taosArrayGetSize(pWal->fileInfoSet);
pWal->writeCur = actualFileNum - 1;
-
+ pWal->totSize = totSize;
+ pWal->vers.lastVer = -1;
if (actualFileNum > 0) {
- int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer;
- if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) {
- fixed = true;
- pWal->vers.lastVer = fLastVer;
- }
- int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
- if (fFirstVer != pWal->vers.firstVer) {
- fixed = true;
- pWal->vers.firstVer = fFirstVer;
- }
+ pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
+ pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer;
+ }
+ (void)walAlignVersions(pWal);
+
+ // update meta file
+ if (updateMeta) {
+ (void)walSaveMeta(pWal);
+ }
+ return 0;
+}
+
+int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
+ if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
}
- if (fixed) {
- walSaveMeta(pWal);
+ if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ if (walValidHeadCksum(pCkHead) != 0) {
+ terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
+ return -1;
}
return 0;
}
-int walCheckAndRepairIdx(SWal* pWal) {
+int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
- for (int32_t i = 0; i < sz; i++) {
- SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
+ ASSERT(fileIdx >= 0 && fileIdx < sz);
+ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
+ char fnameStr[WAL_FILE_LEN];
+ walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
+ char fLogNameStr[WAL_FILE_LEN];
+ walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
+ int64_t fileSize = 0;
- char fnameStr[WAL_FILE_LEN];
- walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
- int64_t fsize;
- TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
- if (pIdxFile == NULL) {
- ASSERT(0);
+ if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) {
+ wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer);
+ if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
+ return 0;
+ }
+
+ // start to repair
+ int64_t offset = fileSize - fileSize % sizeof(SWalIdxEntry);
+ TdFilePtr pLogFile = NULL;
+ TdFilePtr pIdxFile = NULL;
+ SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)};
+ SWalCkHead ckHead;
+ memset(&ckHead, 0, sizeof(ckHead));
+ ckHead.head.version = idxEntry.ver;
+
+ pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
+ if (pIdxFile == NULL) {
+ wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
+ if (pLogFile == NULL) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
+ goto _err;
+ }
+
+ // determine the last valid entry end, i.e. offset
+ while ((offset -= sizeof(SWalIdxEntry)) >= 0) {
+ if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
+ wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr());
- return -1;
+ goto _err;
}
- taosFStatFile(pIdxFile, &fsize, NULL);
- if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
- taosCloseFile(&pIdxFile);
+ if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
+ wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ offset, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (idxEntry.ver > pFileInfo->lastVer) {
continue;
}
- int32_t left = fsize % sizeof(SWalIdxEntry);
- int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END);
- if (left != 0) {
- taosFtruncateFile(pIdxFile, offset);
- wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr,
- offset, fsize);
- }
- offset -= sizeof(SWalIdxEntry);
-
- SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer};
- while (1) {
- if (offset < 0) {
- taosLSeekFile(pIdxFile, 0, SEEK_SET);
- taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
- break;
- }
- if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d cannot seek offset %ld when repair idx since %s", pWal->cfg.vgId, offset, terrstr());
- }
- int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
- if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
- if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) {
- taosFtruncateFile(pIdxFile, offset);
- wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld",
- pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset);
- offset -= sizeof(SWalIdxEntry);
- } else {
- break;
- }
+ if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
+ wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "",
+ pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver);
+ continue;
}
- if (idxEntry.ver < pFileInfo->lastVer) {
- char fLogNameStr[WAL_FILE_LEN];
- walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
- TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
- if (pLogFile == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
- return -1;
- }
- while (idxEntry.ver < pFileInfo->lastVer) {
- if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset,
- terrstr());
- return -1;
- }
- SWalCkHead ckHead;
- taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead));
- if (idxEntry.ver != ckHead.head.version) {
- // todo truncate this idx also
- taosCloseFile(&pLogFile);
- wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId,
- idxEntry.offset, idxEntry.ver, ckHead.head.version);
- return -1;
- }
- idxEntry.ver = ckHead.head.version + 1;
- idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen;
- wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset);
- taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
- }
- taosCloseFile(&pLogFile);
+ if (idxEntry.ver == ckHead.head.version) {
+ break;
+ }
+ }
+ offset += sizeof(SWalIdxEntry);
+
+ // ftruncate idx file
+ if (offset < fileSize) {
+ if (taosFtruncateFile(pIdxFile, offset) < 0) {
+ wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
+ strerror(errno), offset, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ }
+
+ // rebuild idx file
+ if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) {
+ wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
+ offset, fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ while (idxEntry.ver < pFileInfo->lastVer) {
+ ASSERT(idxEntry.ver == ckHead.head.version);
+
+ idxEntry.ver += 1;
+ idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
+
+ if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
+ wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
+ idxEntry.offset, fLogNameStr);
+ goto _err;
+ }
+ wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset);
+ if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) {
+ wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
+ goto _err;
+ }
+ }
+
+ if (taosFsyncFile(pIdxFile) < 0) {
+ wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
+ goto _err;
+ }
+
+ (void)taosCloseFile(&pLogFile);
+ (void)taosCloseFile(&pIdxFile);
+ return 0;
+
+_err:
+ (void)taosCloseFile(&pLogFile);
+ (void)taosCloseFile(&pIdxFile);
+ return -1;
+}
+
+int walCheckAndRepairIdx(SWal* pWal) {
+ int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+ int32_t fileIdx = sz;
+ while (--fileIdx >= 0) {
+ if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) {
+ wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx);
+ return -1;
}
- taosCloseFile(&pIdxFile);
}
return 0;
}
@@ -495,14 +644,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0);
cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField;
pRoot = cJSON_Parse(bytes);
+ if (!pRoot) goto _err;
pMeta = cJSON_GetObjectItem(pRoot, "meta");
+ if (!pMeta) goto _err;
pField = cJSON_GetObjectItem(pMeta, "firstVer");
+ if (!pField) goto _err;
pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pMeta, "snapshotVer");
+ if (!pField) goto _err;
pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pMeta, "commitVer");
+ if (!pField) goto _err;
pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pMeta, "lastVer");
+ if (!pField) goto _err;
pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField));
pFiles = cJSON_GetObjectItem(pRoot, "files");
@@ -512,17 +667,23 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
taosArrayEnsureCap(pArray, sz);
SWalFileInfo* pData = pArray->pData;
for (int i = 0; i < sz; i++) {
- cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
+ cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
+ if (!pInfoJson) goto _err;
SWalFileInfo* pInfo = &pData[i];
pField = cJSON_GetObjectItem(pInfoJson, "firstVer");
+ if (!pField) goto _err;
pInfo->firstVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "lastVer");
+ if (!pField) goto _err;
pInfo->lastVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "createTs");
+ if (!pField) goto _err;
pInfo->createTs = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "closeTs");
+ if (!pField) goto _err;
pInfo->closeTs = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "fileSize");
+ if (!pField) goto _err;
pInfo->fileSize = atoll(cJSON_GetStringValue(pField));
}
taosArraySetSize(pArray, sz);
@@ -530,6 +691,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
pWal->writeCur = sz - 1;
cJSON_Delete(pRoot);
return 0;
+
+_err:
+ cJSON_Delete(pRoot);
+ return -1;
}
static int walFindCurMetaVer(SWal* pWal) {
@@ -565,22 +730,63 @@ static int walFindCurMetaVer(SWal* pWal) {
int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
- walBuildMetaName(pWal, metaVer + 1, fnameStr);
- TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE);
- if (pMetaFile == NULL) {
+ char tmpFnameStr[WAL_FILE_LEN];
+ int n;
+
+ // fsync the idx and log file at first to ensure validity of meta
+ if (taosFsyncFile(pWal->pIdxFile) < 0) {
+ wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
+
+ if (taosFsyncFile(pWal->pLogFile) < 0) {
+ wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ // flush to a tmpfile
+ n = walBuildTmpMetaName(pWal, tmpFnameStr);
+ ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name");
+
+ TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+ if (pMetaFile == NULL) {
+ wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
if (len != taosWriteFile(pMetaFile, serialized, len)) {
- // TODO:clean file
-
- taosCloseFile(&pMetaFile);
- taosRemoveFile(fnameStr);
- return -1;
+ wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (taosFsyncFile(pMetaFile) < 0) {
+ wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ if (taosCloseFile(&pMetaFile) < 0) {
+ wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ // rename it
+ n = walBuildMetaName(pWal, metaVer + 1, fnameStr);
+ ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name");
+
+ if (taosRenameFile(tmpFnameStr, fnameStr) < 0) {
+ wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
}
- taosCloseFile(&pMetaFile);
// delete old file
if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr);
@@ -588,6 +794,11 @@ int walSaveMeta(SWal* pWal) {
}
taosMemoryFree(serialized);
return 0;
+
+_err:
+ taosCloseFile(&pMetaFile);
+ taosMemoryFree(serialized);
+ return -1;
}
int walLoadMeta(SWal* pWal) {
@@ -629,6 +840,10 @@ int walLoadMeta(SWal* pWal) {
}
// load into fileInfoSet
int code = walMetaDeserialize(pWal, buf);
+ if (code < 0) {
+ wError("failed to deserialize wal meta. file:%s", fnameStr);
+ terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+ }
taosCloseFile(&pFile);
taosMemoryFree(buf);
return code;
diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c
index 7974f3e32e..29058cada1 100644
--- a/source/libs/wal/src/walMgmt.c
+++ b/source/libs/wal/src/walMgmt.c
@@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
return NULL;
}
+ if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ taosMemoryFree(pWal);
+ return NULL;
+ }
+
// set config
memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg));
@@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
tstrncpy(pWal->path, path, sizeof(pWal->path));
if (taosMkDir(pWal->path) != 0) {
wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
- taosMemoryFree(pWal);
- return NULL;
+ goto _err;
}
// init ref
pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
if (pWal->pRefHash == NULL) {
- taosMemoryFree(pWal);
- return NULL;
+ wError("failed to init hash since %s", tstrerror(terrno));
+ goto _err;
}
// open meta
@@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
if (pWal->fileInfoSet == NULL) {
wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
- taosHashCleanup(pWal->pRefHash);
- taosMemoryFree(pWal);
- return NULL;
+ goto _err;
}
// init status
@@ -131,46 +134,37 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
pWal->writeHead.head.protoVer = WAL_PROTO_VER;
pWal->writeHead.magic = WAL_MAGIC;
- if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) {
- taosArrayDestroy(pWal->fileInfoSet);
- taosHashCleanup(pWal->pRefHash);
- taosMemoryFree(pWal);
- return NULL;
- }
-
- pWal->refId = taosAddRef(tsWal.refSetId, pWal);
- if (pWal->refId < 0) {
- taosHashCleanup(pWal->pRefHash);
- taosThreadMutexDestroy(&pWal->mutex);
- taosArrayDestroy(pWal->fileInfoSet);
- taosMemoryFree(pWal);
- return NULL;
- }
-
- walLoadMeta(pWal);
+ // load meta
+ (void)walLoadMeta(pWal);
if (walCheckAndRepairMeta(pWal) < 0) {
wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId);
- taosHashCleanup(pWal->pRefHash);
- taosRemoveRef(tsWal.refSetId, pWal->refId);
- taosThreadMutexDestroy(&pWal->mutex);
- taosArrayDestroy(pWal->fileInfoSet);
- return NULL;
+ goto _err;
}
if (walCheckAndRepairIdx(pWal) < 0) {
wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId);
- taosHashCleanup(pWal->pRefHash);
- taosRemoveRef(tsWal.refSetId, pWal->refId);
- taosThreadMutexDestroy(&pWal->mutex);
- taosArrayDestroy(pWal->fileInfoSet);
- return NULL;
+ goto _err;
+ }
+
+ // add ref
+ pWal->refId = taosAddRef(tsWal.refSetId, pWal);
+ if (pWal->refId < 0) {
+ wError("failed to add ref for Wal since %s", tstrerror(terrno));
+ goto _err;
}
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,
pWal->cfg.fsyncPeriod);
-
return pWal;
+
+_err:
+ taosArrayDestroy(pWal->fileInfoSet);
+ taosHashCleanup(pWal->pRefHash);
+ taosThreadMutexDestroy(&pWal->mutex);
+ taosMemoryFree(pWal);
+ pWal = NULL;
+ return NULL;
}
int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
@@ -195,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
void walClose(SWal *pWal) {
taosThreadMutexLock(&pWal->mutex);
+ (void)walSaveMeta(pWal);
taosCloseFile(&pWal->pLogFile);
pWal->pLogFile = NULL;
taosCloseFile(&pWal->pIdxFile);
pWal->pIdxFile = NULL;
- walSaveMeta(pWal);
taosArrayDestroy(pWal->fileInfoSet);
pWal->fileInfoSet = NULL;
taosHashCleanup(pWal->pRefHash);
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index b2cd4fac11..179d809c84 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
- ASSERT(pRet != NULL);
+ if (pRet == NULL) {
+ wError("failed to find WAL log file with ver:%lld", ver);
+ terrno = TSDB_CODE_WAL_INVALID_VER;
+ return -1;
+ }
if (pReader->curFileFirstVer != pRet->firstVer) {
// error code was set inner
if (walReadChangeFile(pReader, pRet->firstVer) < 0) {
@@ -472,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
- ASSERT(0);
+ wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s",
+ pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}
@@ -505,6 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
+ wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s",
+ pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}
diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c
index 1196914dae..4b75db52b7 100644
--- a/source/libs/wal/src/walSeek.c
+++ b/source/libs/wal/src/walSeek.c
@@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->pLogFile != NULL) {
+ code = taosFsyncFile(pWal->pLogFile);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
}
}
if (pWal->pIdxFile != NULL) {
+ code = taosFsyncFile(pWal->pIdxFile);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index bb6c3a91ef..91fa49fce0 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -221,10 +221,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
taosCloseFile(&pIdxFile);
taosCloseFile(&pLogFile);
- taosFsyncFile(pWal->pLogFile);
- taosFsyncFile(pWal->pIdxFile);
-
- walSaveMeta(pWal);
+ code = walSaveMeta(pWal);
+ if (code < 0) {
+ wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
+ taosThreadMutexUnlock(&pWal->mutex);
+ return -1;
+ }
// unlock
taosThreadMutexUnlock(&pWal->mutex);
@@ -396,7 +398,11 @@ int32_t walRollImpl(SWal *pWal) {
pWal->lastRollSeq = walGetSeq();
- walSaveMeta(pWal);
+ code = walSaveMeta(pWal);
+ if (code < 0) {
+ wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
+ goto END;
+ }
END:
return code;
@@ -552,6 +558,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
void walFsync(SWal *pWal, bool forceFsync) {
if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
+ wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
+ if (taosFsyncFile(pWal->pIdxFile) < 0) {
+ wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
+ strerror(errno));
+ }
wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
if (taosFsyncFile(pWal->pLogFile) < 0) {
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c
index f0442c6fd1..616ab7875d 100644
--- a/source/os/src/osEnv.c
+++ b/source/os/src/osEnv.c
@@ -87,11 +87,17 @@ void osUpdate() {
void osCleanup() {}
-bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; }
+bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; }
-bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; }
+bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; }
-bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; }
+bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; }
+
+bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; }
+
+bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; }
+
+bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; }
void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); }
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index eb13a08be4..9a117c6eb4 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -447,12 +447,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f
TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset")
// wal
-TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Wal unexpected generic error")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch")
// tfs
TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config")
diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
index 0a835ec564..c68375894c 100644
--- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
+++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py
@@ -32,6 +32,10 @@ class TDTestCase:
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
+ elif "src" in selfPath:
+ projPath = selfPath[: selfPath.find("src")]
+ elif "/tools/" in selfPath:
+ projPath = selfPath[: selfPath.find("/tools/")]
else:
projPath = selfPath[: selfPath.find("tests")]
@@ -55,11 +59,10 @@ class TDTestCase:
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
tdSql.execute("reset query cache")
- tdSql.query("select count(*) from (select distinct(tbname) from db.stb)")
- tdSql.checkData(0, 0, 10)
+ tdSql.query("show db.tables")
+ tdSql.checkRows(10)
tdSql.query("select count(*) from db.stb")
- if len(tdSql.queryResult):
- tdLog.exit("query result is %d" % len(tdSql.queryResult))
+ tdSql.checkData(0, 0, 100)
def stop(self):
tdSql.close()
diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
index d4b2aae2fb..da22ef75e2 100644
--- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
+++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json
@@ -20,8 +20,10 @@
"super_tables": [{
"name": "stb",
"childtable_prefix": "stb_",
+ "childtable_count": 10,
+ "insert_rows": 10,
"columns": [{"type": "INT"}],
"tags": [{"type": "INT"}]
}]
}]
-}
\ No newline at end of file
+}
diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh
new file mode 100755
index 0000000000..457c9eae20
--- /dev/null
+++ b/tests/script/coverage_test.sh
@@ -0,0 +1,291 @@
+#!/bin/bash
+
+branch=
+if [ x$1 != x ];then
+ branch=$1
+ echo "Testing branch: $branch"
+else
+ echo "Please enter branch name as a parameter"
+ exit 1
+fi
+
+today=`date +"%Y%m%d"`
+TDENGINE_DIR=/root/pxiao/TDengine
+TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDengine {
+ echo "check if TDengine need build"
+ cd $TDENGINE_DIR
+ git remote prune origin > /dev/null
+ git remote update > /dev/null
+ REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch`
+ LOCAL_COMMIT=`git rev-parse --short @`
+ echo " LOCAL: $LOCAL_COMMIT"
+ echo "REMOTE: $REMOTE_COMMIT"
+
+ # reset counter
+ lcov -d . --zerocounters
+
+
+ if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+ echo "repo up-to-date"
+ else
+ echo "repo need to pull"
+ fi
+
+ git reset --hard
+ git checkout -- .
+ git checkout $branch
+ git clean -dfx
+ git pull
+ git submodule update --init --recursive -f
+
+ [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
+ cd $TDENGINE_DIR/debug
+
+ echo "rebuild.."
+ LOCAL_COMMIT=`git rev-parse --short @`
+
+ rm -rf *
+ if [ "$branch" == "3.0" ]; then
+ echo "3.0 ============="
+ cmake -DCOVER=true -DBUILD_TEST=true ..
+ else
+ cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null
+ fi
+ make -j4
+ make install
+}
+
+function runGeneralCaseOneByOne {
+ while read -r line; do
+ if [[ $line =~ ^./test.sh* ]]; then
+ case=`echo $line | grep sim$ | awk '{print $NF}'`
+
+ if [ -n "$case" ]; then
+ date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \
+ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \
+ || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+ fi
+ fi
+ done < $1
+}
+
+function runTestNGCaseOneByOne {
+ while read -r line; do
+ if [[ $line =~ ^taostest* ]]; then
+ case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2`
+ yaml=`echo $line | cut -d' ' -f 2`
+
+ if [ -n "$case" ]; then
+ date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \
+ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \
+ || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+ fi
+ fi
+ done < $1
+}
+
+function runTest {
+ echo "run Test"
+
+ if [ "$branch" == "3.0" ]; then
+ echo "start run unit test case ................"
+ echo " $TDENGINE_DIR/debug "
+ cd $TDENGINE_DIR/debug
+ ctest -j12
+ echo "3.0 unit test done"
+ fi
+
+ cd $TDENGINE_DIR/tests/script
+
+ [ -d ../../sim ] && rm -rf ../../sim
+ [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT
+
+ runGeneralCaseOneByOne jenkins/basic.txt
+
+ sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT
+
+ totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l`
+ if [ "$totalSuccess" -gt "0" ]; then
+ sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+
+ totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l`
+ if [ "$totalFailed" -ne "0" ]; then
+ sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+ sed "3G" $TDENGINE_COVERAGE_REPORT
+
+ stopTaosd
+ echo "run TestNG cases"
+ rm -rf /var/lib/taos/*
+ rm -rf /var/log/taos/*
+ nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
+ sleep 10
+ cd $TDENGINE_DIR/../TestNG/cases
+ runTestNGCaseOneByOne ../scripts/cases.txt
+ echo "TestNG cases done"
+
+ cd $TDENGINE_DIR/tests
+ rm -rf ../sim
+ /root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT
+
+
+ sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT
+ totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
+ if [ "$totalPySuccess" -gt "0" ]; then
+ sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+
+ totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'`
+ if [ -z $totalPyFailed ]; then
+ sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT
+ else
+ sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT
+ fi
+
+ echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT
+ # Test Connector
+ stopTaosd
+ nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 &
+ sleep 10
+
+ cd $TDENGINE_DIR/src/connector/jdbc
+ mvn clean package > /dev/null 2>&1
+ mvn test > jdbc-out.log 2>&1
+ tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+ # Test C Demo
+ stopTaosd
+ $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
+ sleep 10
+ yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+ # Test waltest
+ dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
+ walDir=`find $dataDir -name "wal"|head -n1`
+ echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
+ echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
+ if [ -n "$walDir" ]; then
+ yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT
+ fi
+
+ # run Unit Test
+ echo "Run Unit Test: utilTest, queryTest and cliTest"
+ #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!"
+ #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!"
+ #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!"
+
+ stopTaosd
+
+ cd $TDENGINE_DIR/tests/script
+ find . -name '*.sql' | xargs rm -f
+
+ cd $TDENGINE_DIR/tests/pytest
+ find . -name '*.sql' | xargs rm -f
+}
+
+function lcovFunc {
+ echo "collect data by lcov"
+ cd $TDENGINE_DIR
+
+ # collect data
+ lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
+
+ # remove exclude paths
+ if [ "$branch" == "3.0" ]; then
+ lcov --remove coverage.info \
+ '*/contrib/*' '*/tests/*' '*/test/*'\
+ '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\
+ '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\
+ '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\
+ '*/tthread.c' '*/tversion.c'\
+ --rc lcov_branch_coverage=1 -o coverage.info
+ else
+ lcov --remove coverage.info \
+ '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\
+ --rc lcov_branch_coverage=1 -o coverage.info
+ fi
+
+
+ # generate result
+ echo "generate result"
+ lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
+
+ # push result to coveralls.io
+ echo "push result to coveralls.io"
+ /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT
+
+ #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT
+ #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT
+ cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT
+
+}
+
+function sendReport {
+ echo "send report"
+ receiver="develop@taosdata.com"
+ mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+ cd $TDENGINE_DIR
+
+ sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT
+ BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT`
+ echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+ (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \
+ /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+ echo "Stop taosd start"
+ systemctl stop taosd
+ PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+ while [ -n "$PID" ]
+ do
+ pkill -TERM -x taosd
+ sleep 1
+ PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+ done
+ echo "Stop tasod end"
+}
+
+function runTestRandomFail {
+ exec_random_fail_sh=$1
+ default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh
+ [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1
+
+ dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py
+ default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py
+ [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1
+
+ runTest NoRandomFail
+}
+
+WORK_DIR=/root/pxiao
+
+date >> $WORK_DIR/cron.log
+echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log
+
+stopTaosd
+buildTDengine
+
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh
+#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh
+
+runTest
+
+lcovFunc
+#sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 89e26e785b..d8f4a36261 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -155,7 +155,7 @@
./test.sh -f tsim/parser/select_with_tags.sim
./test.sh -f tsim/parser/selectResNum.sim
./test.sh -f tsim/parser/set_tag_vals.sim
-./test.sh -f tsim/parser/single_row_in_tb.sim
+# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim
./test.sh -f tsim/parser/sliding.sim
./test.sh -f tsim/parser/slimit_alter_tags.sim
./test.sh -f tsim/parser/slimit.sim
@@ -194,7 +194,7 @@
./test.sh -f tsim/mnode/basic1.sim
./test.sh -f tsim/mnode/basic2.sim
./test.sh -f tsim/mnode/basic3.sim
-# TD-17919 ./test.sh -f tsim/mnode/basic4.sim
+./test.sh -f tsim/mnode/basic4.sim
./test.sh -f tsim/mnode/basic5.sim
# ---- show ----
diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim
index 59156080c8..31fc276298 100644
--- a/tests/script/tsim/mnode/basic1.sim
+++ b/tests/script/tsim/mnode/basic1.sim
@@ -42,6 +42,7 @@ sql_error drop mnode on dnode 1
print =============== create mnode 2
sql create mnode on dnode 2
+print =============== create mnode 2 finished
$x = 0
step2:
$x = $x + 1
@@ -69,9 +70,10 @@ if $data(2)[2] != follower then
goto step2
endi
-sleep 2000
print ============ drop mnode 2
sql drop mnode on dnode 2
+
+print ============ drop mnode 2 finished
sql select * from information_schema.ins_mnodes
if $rows != 1 then
return -1
@@ -109,6 +111,8 @@ sleep 2000
print =============== create mnodes
sql create mnode on dnode 2
+
+print =============== create mnode 2 finished
sql select * from information_schema.ins_mnodes
if $rows != 2 then
return -1
diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim
index a2b8aa2f96..6739a87b42 100644
--- a/tests/script/tsim/mnode/basic4.sim
+++ b/tests/script/tsim/mnode/basic4.sim
@@ -45,49 +45,12 @@ endi
if $data(2)[4] != ready then
goto step2
endi
-
-system sh/exec.sh -n dnode3 -s stop
-sql_error create mnode on dnode 3
-
-print =============== step3: select * from information_schema.ins_mnodes
-
-$x = 0
-step3:
- $x = $x + 1
- sleep 1000
- if $x == 10 then
- return -1
- endi
-sql select * from information_schema.ins_mnodes -x step3
-print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4]
-print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4]
-
-$leaderNum = 0
-if $data(1)[2] == leader then
- $leaderNum = 1
-endi
-if $data(2)[2] == leader then
- $leaderNum = 1
-endi
-if $leaderNum == 0 then
- goto step3
-endi
-if $data(3)[2] != offline then
- goto step3
-endi
-if $data(1)[3] != ready then
- goto step3
-endi
-if $data(2)[3] != ready then
- goto step3
-endi
-if $data(3)[3] != creating then
- goto step3
+if $data(3)[4] != ready then
+ goto step2
endi
-print =============== step4: start dnode3
-system sh/exec.sh -n dnode3 -s start
+print =============== step4: create mnode 3
+sql create mnode on dnode 3
$x = 0
step4:
@@ -159,12 +122,11 @@ endi
if $data(2)[3] != ready then
goto step5
endi
-if $data(3)[3] != dropping then
- goto step5
-endi
print =============== step6: start dnode3
system sh/exec.sh -n dnode3 -s start
+sql drop mnode on dnode 1 -x step60
+step60:
$x = 0
step6:
diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim
index 387f38a717..16e8fa3dfa 100644
--- a/tests/script/tsim/mnode/basic5.sim
+++ b/tests/script/tsim/mnode/basic5.sim
@@ -232,12 +232,11 @@ endi
if $leaderNum != 1 then
goto step81
endi
-if $data(1)[3] != dropping then
- goto step81
-endi
print =============== step9: start mnode1 and wait it dropped
system sh/exec.sh -n dnode1 -s start
+sql drop mnode on dnode 1 -x step90
+step90:
$x = 0
step91:
diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim
index 6fa7e39335..252ef10b1a 100644
--- a/tests/script/tsim/valgrind/checkError3.sim
+++ b/tests/script/tsim/valgrind/checkError3.sim
@@ -62,7 +62,16 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start -v
print =============== stepa: query data
-sql select * from c1
+
+$x = 0
+steps:
+ $x = $x + 1
+ sleep 500
+ if $x == 50 then
+ return -1
+ endi
+sql select * from c1 -x steps
+
sql select * from stb
sql select * from stb_1
sql select ts, c1, c2, c3 from c1
diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim
index 11a387ed4d..89b6cadef8 100644
--- a/tests/script/tsim/valgrind/checkError6.sim
+++ b/tests/script/tsim/valgrind/checkError6.sim
@@ -158,9 +158,15 @@ print =============== restart
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start -v
-sleep 1000
+$x = 0
+steps:
+ $x = $x + 1
+ sleep 500
+ if $x == 50 then
+ return -1
+ endi
+sql select avg(tbcol) as c from stb -x steps
-sql select avg(tbcol) as c from stb
sql select avg(tbcol) as c from stb where ts <= 1601481840000
sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000
sql select avg(tbcol) as c from stb interval(1m)
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 4c823faa8a..cc85b8ddc8 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
-# unstable python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3
@@ -248,8 +248,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 -
python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5
# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py
-# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
-# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3
+python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
+python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3
python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3
diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c
index 8aafd747af..46eaf6cff1 100644
--- a/tools/shell/src/shellAuto.c
+++ b/tools/shell/src/shellAuto.c
@@ -325,24 +325,23 @@ int cntDel = 0; // delete byte count after next press tab
// show auto tab introduction
void printfIntroduction() {
- printf(" **************************** How To Use TAB Key ********************************\n");
- printf(" * TDengine Command Line supports pressing TAB key to complete word, *\n");
- printf(" * including database name, table name, function name and keywords. *\n");
- printf(" * Press TAB key anywhere, you'll get surprise. *\n");
- printf(" * KEYBOARD SHORTCUT: *\n");
- printf(" * [ TAB ] ...... Complete the word or show help if no input *\n");
- printf(" * [ Ctrl + A ] ...... move cursor to [A]head of line *\n");
- printf(" * [ Ctrl + E ] ...... move cursor to [E]nd of line *\n");
- printf(" * [ Ctrl + W ] ...... move cursor to line of middle *\n");
- printf(" * [ Ctrl + L ] ...... clean screen *\n");
- printf(" * [ Ctrl + K ] ...... clean after cursor *\n");
- printf(" * [ Ctrl + U ] ...... clean before cursor *\n");
- printf(" * *\n");
+ printf(" ****************************** Tab Completion **********************************\n");
+ printf(" * The TDengine CLI supports tab completion for a variety of items, *\n");
+ printf(" * including database names, table names, function names and keywords. *\n");
+ printf(" * The full list of shortcut keys is as follows: *\n");
+ printf(" * [ TAB ] ...... complete the current word *\n");
+ printf(" * ...... if used on a blank line, display all valid commands *\n");
+ printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n");
+ printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n");
+ printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n");
+ printf(" * [ Ctrl + L ] ...... clear the entire screen *\n");
+ printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n");
+ printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n");
printf(" **********************************************************************************\n\n");
}
void showHelp() {
- printf("\nThe following are supported commands for TDengine Command Line:");
+ printf("\nThe TDengine CLI supports the following commands:");
printf(
"\n\
----- A ----- \n\
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index fd0ea60323..37d00c0a7c 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) {
printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6);
// call auto tab
- callbackAutoTab(command, pSql, false);
+ callbackAutoTab(command, NULL, false);
}
printf("\r\n");
@@ -998,7 +998,9 @@ void *shellCancelHandler(void *arg) {
shell.stop_query = true;
} else {
#endif
- taos_kill_query(shell.conn);
+ if (shell.conn) {
+ taos_kill_query(shell.conn);
+ }
#ifdef WEBSOCKET
}
#endif