diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 7bde332c8c..f0a1ad83ad 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d58230c + GIT_TAG 2849aa4 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/12-taos-sql/28-recovery.md b/docs/en/12-taos-sql/28-recovery.md index 14ac14f867..e869ffc45f 100644 --- a/docs/en/12-taos-sql/28-recovery.md +++ b/docs/en/12-taos-sql/28-recovery.md @@ -16,10 +16,10 @@ You can use the SHOW CONNECTIONS statement to find the conn_id. ## Terminate a Query ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -You can use the SHOW QUERIES statement to find the query_id. +You can use the SHOW QUERIES statement to find the kill_id. ## Terminate a Transaction diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index d95086d4c4..6295d8553d 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js - **name**: specify the name of the database. -- **drop**: indicate whether to delete the database before inserting. The default is true. +- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop. #### Stream processing related configuration parameters diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 4c33b5232a..9346c3f763 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -340,7 +340,7 @@ LTRIM(expr) #### RTRIM ```sql -LTRIM(expr) +RTRIM(expr) ``` **功能说明**：返回清除右边空格后的字符串。 diff --git a/docs/zh/12-taos-sql/28-recovery.md b/docs/zh/12-taos-sql/28-recovery.md index 582c373907..b5088e7982 100644 --- a/docs/zh/12-taos-sql/28-recovery.md +++ b/docs/zh/12-taos-sql/28-recovery.md @@ -17,10 +17,10 @@ conn_id 可以通过 `SHOW CONNECTIONS` 获取。 ## 终止查询 ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -query_id 可以通过 `SHOW QUERIES` 获取。 +kill_id 可以通过 `SHOW QUERIES` 获取。 ## 终止事务 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 7da8e9f331..1ab7f4f016 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 + + +## SCHEMALESS 变更 + +| # | **元素** | **
差异性
** | **说明** | +| - | :------- | :-------- | :------- | +| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。 diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 9f4f728f78..e067c646b2 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **name** : 数据库名。 -- **drop** : 插入前是否删除数据库，默认为 true。 +- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。 #### 流式计算相关配置参数 diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 07d5b0692b..b89f625376 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -115,7 +115,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 ## 数据模式映射规则 -本节将说明行协议的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 +本节将说明 InfluxDB 行协议(Line Protocol)的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 超级表名称。tag_set 中的 标签名称为 数据模式中的标签名，field_set 中的名称为列名称。以如下数据为例，说明映射规则： ```json diff --git a/examples/c/demoapi.c b/examples/c/demoapi.c index c3a6d26776..18f1b5a059 100644 --- a/examples/c/demoapi.c +++ b/examples/c/demoapi.c @@ -1,5 +1,17 @@ -// C api call sequence demo -// to compile: gcc -o apidemo apidemo.c -ltaos +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include #include @@ -13,23 +25,23 @@ #include "taos.h" #define debugPrint(fmt, ...) \ - do { if (g_args.debug_print || g_args.verbose_print) \ - fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0) + do { if (g_args.debug_print || g_args.verbose_print) {\ + fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0) #define warnPrint(fmt, ...) \ do { fprintf(stderr, "\033[33m"); \ fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define errorPrint(fmt, ...) \ do { fprintf(stderr, "\033[31m"); \ fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define okPrint(fmt, ...) \ do { fprintf(stderr, "\033[32m"); \ fprintf(stderr, "OK: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) int64_t g_num_of_tb = 2; int64_t g_num_of_rec = 3; @@ -74,10 +86,16 @@ static void prepare_data(TAOS* taos) { res = taos_query(taos, "create database test;"); taos_free_result(res); taosMsleep(100); - taos_select_db(taos, "test"); + if (taos_select_db(taos, "test")) { + errorPrint("%s() LN%d: taos_select_db() failed\n", + __func__, __LINE__); + return; + } char command[1024] = {0}; - sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));"); + sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, " + "bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, " + "city binary(20), dist nchar(20), street binary(20));"); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { okPrint("%s created\n", "meters"); @@ -117,12 +135,11 @@ static void prepare_data(TAOS* taos) { sprintf(command, "insert into t%"PRId64" " "values(%" PRId64 ", %f, %"PRId64", " "'%c%d', '%s%c%d', '%c%d')", - i, 1650000000000+j, (float)j, j, + i, 1650000000000+j, j * 1.0, j, 'a'+(int)j%25, rand(), // "涛思", 'z' - (int)j%25, rand(), "TAOS", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -150,8 +167,7 @@ static void prepare_data(TAOS* taos) { i, 1650000000000+j+1, (float)j, j, 'a'+(int)j%25, rand(), "数据", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -179,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { } if (block) { - warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n"); + warnPrint("%s", "call taos_fetch_block(), " + "don't call taos_fetch_lengths()\n"); int rows = 0; while ((rows = taos_fetch_block(res, &row))) { int *lengths = taos_fetch_lengths(res); @@ -190,7 +207,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d type is %d, no need get offset\n", f, fields[f].type); for (int64_t c = 0; c < rows; c++) { - switch(fields[f].type) { + switch (fields[f].type) { case TSDB_DATA_TYPE_TIMESTAMP: if (taos_is_null(res, c, f)) { printf("col%d, row: %"PRId64" " @@ -199,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %"PRId64"\n", f, c, - *(int64_t*)((char*)(row[f])+c*sizeof(int64_t))); + *(int64_t*)((char*)(row[f]) + +c*sizeof(int64_t))); } break; @@ -211,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %d\n", f, c, - *(int32_t*)((char*)(row[f])+c*sizeof(int32_t))); + *(int32_t*)((char*)(row[f]) + +c*sizeof(int32_t))); } break; @@ -223,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %f\n", f, c, - *(float*)((char*)(row[f])+c*sizeof(float))); + *(float*)((char*)(row[f]) + +c*sizeof(float))); } break; @@ -238,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { if (offsets) { for (int c = 0; c < rows; c++) { if (offsets[c] != -1) { - int length = *(int16_t*)((char*)(row[f]) + offsets[c]); + int length = *(int16_t*)((char*)(row[f]) + + offsets[c]); char *buf = calloc(1, length + 1); - strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length); - printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n", + strncpy(buf, (char *)((char*)(row[f]) + + offsets[c] + 2), length); + printf("row: %d, col: %d, offset: %d, " + "length: %d, content: %s\n", c, f, offsets[c], length, buf); free(buf); } else { - printf("row: %d, col: %d, offset: -1, means content is NULL\n", + printf("row: %d, col: %d, offset: -1, " + "means content is NULL\n", c, f); } } @@ -267,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { int* lengths = taos_fetch_lengths(res); if (lengths) { for (int c = 0; c < num_fields; c++) { - printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n", + printf("row: %"PRId64", col: %d, is_null: %s, " + "length of column %d is %d\n", num_rows, c, taos_is_null(res, num_rows, c)?"True":"False", c, lengths[c]); @@ -277,7 +302,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { __func__, __LINE__, tbname); } - num_rows ++; + num_rows++; } } @@ -322,7 +347,8 @@ int main(int argc, char *argv[]) { #endif TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { - printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); + printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", + taos_errstr(taos)); exit(1); } diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 0e22985059..fd0521af3b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -93,6 +93,7 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in // query client extern int32_t tsQueryPolicy; +extern int32_t tsQueryRspPolicy; extern int32_t tsQuerySmaOptimize; extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 0d43539629..cdb1642a5c 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -30,9 +30,10 @@ typedef struct SMnode SMnode; typedef struct { int32_t dnodeId; - bool standby; bool deploy; - SReplica replica; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; SMsgCb msgCb; } SMnodeOpt; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 777fe1c4df..7d95e840e9 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -54,6 +54,9 @@ typedef enum { #define QUERY_POLICY_QNODE 3 #define QUERY_POLICY_CLIENT 4 +#define QUERY_RSP_POLICY_DELAY 0 +#define QUERY_RSP_POLICY_QUICK 1 + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 411a63a379..0f8220f19c 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -211,7 +211,7 @@ typedef struct SSyncInfo { int32_t syncInit(); void syncCleanUp(); -int64_t syncOpen(const SSyncInfo* pSyncInfo); +int64_t syncOpen(SSyncInfo* pSyncInfo); void syncStart(int64_t rid); void syncStop(int64_t rid); int32_t syncSetStandby(int64_t rid); @@ -233,7 +233,7 @@ const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot); -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg); // build SRpcMsg, need to call syncPropose with SRpcMsg int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); diff --git a/include/util/trbtree.h b/include/util/trbtree.h index f6d37e3d75..7c93c8dd8d 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -26,7 +26,7 @@ typedef struct SRBTree SRBTree; typedef struct SRBTreeNode SRBTreeNode; typedef struct SRBTreeIter SRBTreeIter; -typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); +typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *); // SRBTree ============================================= #define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min) @@ -36,7 +36,7 @@ void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn); SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z); void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode); // SRBTreeIter ============================================= #define tRBTreeIterCreate(tree, ascend) \ @@ -53,8 +53,6 @@ struct SRBTreeNode { SRBTreeNode *right; }; -#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) - struct SRBTree { tRBTreeCmprFn cmprFn; int64_t n; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7b437e4769..940dd745c8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1036,30 +1036,38 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->type = pQuery->msgType; SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); + SPlanContext cxt = {.queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite, - .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, - .pUser = pRequest->pTscObj->user, - .sysInfo = pRequest->pTscObj->sysInfo, - .allocatorId = pRequest->allocatorRefId}; - SQueryPlan* pDag = NULL; - int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); - if (code) { - tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); - } else { - pRequest->body.subplanNum = pDag->numOfSubplans; + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user, + .sysInfo = pRequest->pTscObj->sysInfo, + .allocatorId = pRequest->allocatorRefId}; + + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SQueryPlan* pDag = NULL; + + int64_t st = taosGetTimestampUs(); + int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); + if (code) { + tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + pRequest->body.subplanNum = pDag->numOfSubplans; + } + + pRequest->metric.planEnd = taosGetTimestampUs(); + if (code == TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self, + (pRequest->metric.planEnd - st)/1000.0, pRequest->requestId); } - - pRequest->metric.planEnd = taosGetTimestampUs(); - - if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { - SArray* pNodeList = NULL; - buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { + SArray* pNodeList = NULL; + buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter, .requestId = pRequest->requestId, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6f8cef7c0d..7d8026f314 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -222,7 +222,9 @@ void taos_kill_query(TAOS *taos) { int64_t rid = *(int64_t *)taos; STscObj *pTscObj = acquireTscObj(rid); - stopAllRequests(pTscObj->pRequests); + if (pTscObj) { + stopAllRequests(pTscObj->pRequests); + } releaseTscObj(rid); } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 66f992f05f..922cab23c8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -972,6 +972,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; + tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); + for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1297,7 +1299,8 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { pParam->code = code; if (code != 0) { - tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); + tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d, code %x", tmq->consumerId, + pParam->async, code); goto END; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69174476a..21a52a4b57 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) { int32_t n = 0; char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t* length = taos_fetch_lengths(pRes); - for (int32_t i = 0; i < numOfFields; ++i) { - printf("(%d):%d ", i, length[i]); - } - printf("\n"); - - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - memset(str, 0, sizeof(str)); +// int32_t* length = taos_fetch_lengths(pRes); +// for(int32_t i = 0; i < numOfFields; ++i) { +// printf("(%d):%d " , i, length[i]); +// } +// printf("\n"); +// +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// memset(str, 0, sizeof(str)); } } @@ -102,17 +102,6 @@ void queryCallback(void* param, void* res, int32_t code) { taos_fetch_raw_block_a(res, fetchCallback, param); } -void queryCallback1(void* param, void* res, int32_t code) { - if (code != TSDB_CODE_SUCCESS) { - printf("failed to execute, reason:%s\n", taos_errstr(res)); - } - - taos_free_result(res); - - printf("exec query:\n"); - taos_query_a(param, "select * from tm1", queryCallback, param); -} - void createNewTable(TAOS* pConn, int32_t index) { char str[1024] = {0}; sprintf(str, "create table tu%d using st2 tags(%d)", index, index); @@ -123,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for (int32_t i = 0; i < 3280; i += 20) { + for(int32_t i = 0; i < 10000; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -141,10 +130,49 @@ void createNewTable(TAOS* pConn, int32_t index) { taos_free_result(p); } } + +void *queryThread(void *arg) { + TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to db, reason:%s", taos_errstr(pConn)); + return NULL; + } + + int64_t el = 0; + + for (int32_t i = 0; i < 5000000; ++i) { + int64_t st = taosGetTimestampUs(); + TAOS_RES* pRes = taos_query(pConn, + "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;"); + if (taos_errno(pRes) != 0) { + printf("failed, reason:%s\n", taos_errstr(pRes)); + } else { + printResult(pRes); + } + + taos_free_result(pRes); + el += (taosGetTimestampUs() - st); + if (i % 1000 == 0 && i != 0) { + printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000)); + } + } + + taos_close(pConn); + return NULL; +} + +static int32_t numOfThreads = 1; } // namespace int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + numOfThreads = atoi(argv[1]); + } + + numOfThreads = TMAX(numOfThreads, 1); + printf("the runing threads is:%d", numOfThreads); + return RUN_ALL_TESTS(); } @@ -664,7 +692,6 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } -#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -676,13 +703,14 @@ TEST(testCase, projection_query_tables) { // } // taos_free_result(pRes); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu"); taos_free_result(pRes); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } + taos_free_result(pRes); pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); @@ -722,6 +750,16 @@ TEST(testCase, projection_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif + +TEST(testCase, tsbs_perf_test) { + TdThread qid[20] = {0}; + + for(int32_t i = 0; i < numOfThreads; ++i) { + taosThreadCreate(&qid[i], NULL, queryThread, NULL); + } + getchar(); +} #if 0 TEST(testCase, projection_query_stables) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 88df54a9ea..56daa4075a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -90,6 +90,7 @@ bool tsSmlDataFormat = false; // query int32_t tsQueryPolicy = 1; +int32_t tsQueryRspPolicy = 0; int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -350,6 +351,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1; if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; @@ -728,6 +730,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; + tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index a4c37dd334..17cbde437b 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -41,8 +41,8 @@ typedef struct SMnodeMgmt { } SMnodeMgmt; // mmFile.c -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); +int32_t mmReadFile(const char *path, SMnodeOpt *pOption); +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption); // mmHandle.c SArray *mmGetMsgHandles(); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 27a35ae17a..c5ddb9f021 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { +int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 4096; @@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { char file[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { code = 0; @@ -50,38 +50,69 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { dError("failed to read %s since deployed not found", file); goto _OVER; } - *pDeployed = deployed->valueint; + pOption->deploy = deployed->valueint; - cJSON *id = cJSON_GetObjectItem(root, "id"); - if (id) { - if (id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); + cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex"); + if (selfIndex) { + if (selfIndex->type != cJSON_Number) { + dError("failed to read %s since selfIndex not found", file); goto _OVER; } - if (pReplica) { - pReplica->id = id->valueint; - } + pOption->selfIndex = selfIndex->valueint; } - cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn"); - if (fqdn) { - if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); + cJSON *replicas = cJSON_GetObjectItem(root, "replicas"); + if (replicas) { + if (replicas->type != cJSON_Array) { + dError("failed to read %s since replicas not found", file); goto _OVER; } - if (pReplica) { - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - } - } - cJSON *port = cJSON_GetObjectItem(root, "port"); - if (port) { - if (port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); + int32_t numOfReplicas = cJSON_GetArraySize(replicas); + if (numOfReplicas <= 0) { + dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas); goto _OVER; } - if (pReplica) { - pReplica->port = (uint16_t)port->valueint; + pOption->numOfReplicas = numOfReplicas; + + for (int32_t i = 0; i < numOfReplicas; ++i) { + SReplica *pReplica = pOption->replicas + i; + + cJSON *replica = cJSON_GetArrayItem(replicas, i); + if (replica == NULL) break; + + cJSON *id = cJSON_GetObjectItem(replica, "id"); + if (id) { + if (id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->id = id->valueint; + } + } + + cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn"); + if (fqdn) { + if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto _OVER; + } + if (pReplica) { + tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + } + } + + cJSON *port = cJSON_GetObjectItem(replica, "port"); + if (port) { + if (port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->port = (uint16_t)port->valueint; + } + } } } @@ -92,18 +123,18 @@ _OVER: if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); if (code == 0) { - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy); } terrno = code; return code; } -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) { +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - if (pReplica != NULL && pReplica->id > 0) { - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port); + if (pOption->deploy && pOption->numOfReplicas > 0) { + len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex); + len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n"); + + for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + const SReplica *pReplica = pOption->replicas + i; + if (pReplica != NULL && pReplica->id > 0) { + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + } + if (i < pOption->numOfReplicas - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }],\n"); + } + } } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy); len += snprintf(content + len, maxLen - len, "}\n"); taosWriteFile(pFile, content, len); @@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) return -1; } - dDebug("successed to write %s, deployed:%d", realfile, deployed); + dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy); return 0; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index ec761e6441..fc45dbf15f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - if (createReq.replica != 1) { + SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1}; + memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (createReq.replicas[i].id == pInput->pData->dnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { terrno = TSDB_CODE_INVALID_OPTION; - dGError("failed to create mnode since %s", terrstr()); + dGError("failed to create mnode since %s, selfIndex is -1", terrstr()); return -1; } - bool deployed = true; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - bool deployed = false; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, NULL, deployed) != 0) { + SMnodeOpt option = {.deploy = false}; + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 49207225a5..e7f71ad420 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) { } static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - if (mmReadFile(&mgmt, NULL, required) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pInput->path, &option) != 0) { return -1; } - if (!(*required)) { + if (!option.deploy) { *required = mmDeployRequired(pInput); + } else { + *required = true; } return 0; } static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { - pOption->standby = false; pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica.id = 1; - pOption->replica.port = tsServerPort; - tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->selfIndex = 0; + pOption->numOfReplicas = 1; + pOption->replicas[0].id = 1; + pOption->replicas[0].port = tsServerPort; + tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } -static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { - pOption->standby = false; +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pReplica->id > 0) { - pOption->standby = true; - pOption->replica = *pReplica; - } } static void mmClose(SMnodeMgmt *pMgmt) { @@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - bool deployed = false; - SReplica replica = {0}; - if (mmReadFile(pMgmt, &replica, &deployed) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pMgmt->path, &option) != 0) { dError("failed to read file since %s", terrstr()); mmClose(pMgmt); return -1; } - SMnodeOpt option = {0}; - if (!deployed) { + if (!option.deploy) { dInfo("mnode start to deploy"); pMgmt->pData->dnodeId = 1; mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); - mmBuildOptionForOpen(pMgmt, &replica, &option); + mmBuildOptionForOpen(pMgmt, &option); } pMgmt->pMnode = mndOpen(pMgmt->path, &option); @@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (!deployed || replica.id > 0) { - deployed = true; - if (mmWriteFile(pMgmt, NULL, deployed) != 0) { + if (option.numOfReplicas > 0) { + option.deploy = true; + option.numOfReplicas = 0; + if (mmWriteFile(pMgmt->path, &option) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index d2b071ec61..849d3ef390 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -72,7 +72,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { mndPostProcessQueryMsg(pMsg); } - dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); + dGTrace("msg:%p is freed, code:%s", pMsg, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 71315a39fd..f55e830a44 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -87,12 +87,13 @@ typedef struct { typedef struct { tsem_t syncSem; int64_t sync; - SReplica replica; int32_t errCode; int32_t transId; SRWLatch lock; - int8_t standby; int8_t leaderTransferFinish; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; } SSyncMgmt; typedef struct { @@ -130,11 +131,10 @@ typedef struct SMnode { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); int64_t mndGenerateUid(const char *name, int32_t len); -int32_t mndAcquireRpcRef(SMnode *pMnode); -void mndReleaseRpcRef(SMnode *pMnode); -void mndSetRestore(SMnode *pMnode, bool restored); -void mndSetStop(SMnode *pMnode); -bool mndGetStop(SMnode *pMnode); +void mndSetRestored(SMnode *pMnode, bool restored); +bool mndGetRestored(SMnode *pMnode); +void mndSetStop(SMnode *pMnode); +bool mndGetStop(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index cb9d70d5ee..993efbcd08 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -24,7 +24,7 @@ extern "C" { int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); -bool mndIsMaster(SMnode *pMnode); +bool mndIsLeader(SMnode *pMnode); int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId); void mndSyncStart(SMnode *pMnode); void mndSyncStop(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6670211d78..3b31873857 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true); if (pShow->pIter == NULL) break; if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9ade39e8e7..fba1fd94d6 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { goto _OVER; } else { pDnode->accessTimes++; - mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); + mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } } @@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } pDnode->lastAccessTime = curMs; + pDnode->accessTimes++; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 275fc76e36..a52ca1bd42 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -43,6 +43,37 @@ #include "mndUser.h" #include "mndVgroup.h" +static inline int32_t mndAcquireRpc(SMnode *pMnode) { + int32_t code = 0; + taosThreadRwlockRdlock(&pMnode->lock); + if (pMnode->stopped) { + terrno = TSDB_CODE_APP_NOT_READY; + code = -1; + } else if (!mndIsLeader(pMnode)) { + code = -1; + } else { +#if 1 + atomic_add_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is acquired, ref:%d", ref); +#endif + } + taosThreadRwlockUnlock(&pMnode->lock); + return code; +} + +static inline void mndReleaseRpc(SMnode *pMnode) { + taosThreadRwlockRdlock(&pMnode->lock); +#if 1 + atomic_sub_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is released, ref:%d", ref); +#endif + taosThreadRwlockUnlock(&pMnode->lock); +} + static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) { static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->msgCb = pOption->msgCb; pMnode->selfDnodeId = pOption->dnodeId; - pMnode->syncMgmt.replica = pOption->replica; - pMnode->syncMgmt.standby = pOption->standby; + pMnode->syncMgmt.selfIndex = pOption->selfIndex; + pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas; + memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas)); } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) { mError("failed to deploy sdb while start mnode"); return -1; } - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } grantReset(pMnode, TSDB_GRANT_ALL, 0); @@ -570,23 +602,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } - if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; + if (mndAcquireRpc(pMsg->info.node) == 0) return 0; if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { return -1; } - SEpSet epSet = {0}; - mndGetMnodeEpSet(pMsg->info.node, &epSet); + SEpSet epSet = {0}; + SMnode *pMnode = pMsg->info.node; + mndGetMnodeEpSet(pMnode, &epSet); const STraceId *trace = &pMsg->info.traceId; - mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); + mDebug( + "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s " + "numOfEps:%d inUse:%d", + pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync), + syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); if (epSet.numOfEps > 0) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { - mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); } int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); @@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); int32_t code = (*fp)(pMsg); - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mGTrace("msg:%p, won't response immediately since in progress", pMsg); @@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) { int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { - if (mndAcquireRpcRef(pMnode) != 0) return -1; + if (mndAcquireRpc(pMnode) != 0) return -1; SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); @@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc)); if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || pStbInfo->stbs == NULL) { - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return -1; } @@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->timeseries_total = INT32_MAX; } - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return 0; } @@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; } -int32_t mndAcquireRpcRef(SMnode *pMnode) { - int32_t code = 0; - taosThreadRwlockRdlock(&pMnode->lock); - if (pMnode->stopped) { - mTrace("mnode not running"); - terrno = TSDB_CODE_APP_NOT_READY; - code = -1; - } else if (!mndIsMaster(pMnode)) { - mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored); - code = -1; - } else { - int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is acquired, ref:%d", ref); - } - taosThreadRwlockUnlock(&pMnode->lock); - return code; -} - -void mndReleaseRpcRef(SMnode *pMnode) { - taosThreadRwlockRdlock(&pMnode->lock); - int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is released, ref:%d", ref); - taosThreadRwlockUnlock(&pMnode->lock); -} - -void mndSetRestore(SMnode *pMnode, bool restored) { +void mndSetRestored(SMnode *pMnode, bool restored) { if (restored) { taosThreadRwlockWrlock(&pMnode->lock); pMnode->restored = true; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d0440359ff..c27241317c 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -36,6 +36,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq); static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); +static void mndReloadSyncConfig(SMnode *pMnode); int32_t mndInitMnode(SMnode *pMnode) { SSdbTable table = { @@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { } pObj->state = TAOS_SYNC_STATE_ERROR; + mndReloadSyncConfig(pSdb->pMnode); return 0; } @@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; + mndReloadSyncConfig(pSdb->pMnode); + return 0; } @@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pIter == NULL) break; if (pObj->id == pMnode->selfDnodeId) { - if (mndIsMaster(pMnode)) { + if (mndIsLeader(pMnode)) { pEpSet->inUse = pEpSet->numOfEps; } else { pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes; @@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pEpSet->numOfEps == 0) { syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); } + + if (pEpSet->inUse >= pEpSet->numOfEps) { + pEpSet->inUse = 0; + } } static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { @@ -274,13 +282,72 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod return 0; } +static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); + + STransAction action = { + .epSet = *pCreateEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_CREATE_MNODE, + .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + +static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); + + STransAction action = { + .epSet = *pAlterEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_MND_ALTER_MNODE, + .acceptableCode = 0, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { + int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq); + + STransAction action = { + .epSet = *pDroprEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_DROP_MNODE, + .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; SDCreateMnodeReq createReq = {0}; - SEpSet alterEpset = {0}; SEpSet createEpset = {0}; while (1) { @@ -288,75 +355,25 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); if (pIter == NULL) break; - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } + createReq.replicas[numOfReplicas].id = pMObj->id; + createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); numOfReplicas++; sdbRelease(pSdb, pMObj); } - alterReq.replica = numOfReplicas + 1; - alterReq.replicas[numOfReplicas].id = pDnode->id; - alterReq.replicas[numOfReplicas].port = pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.numOfEps = numOfReplicas + 1; - alterEpset.eps[numOfReplicas].port = pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - createReq.replica = 1; - createReq.replicas[0].id = pDnode->id; - createReq.replicas[0].port = pDnode->port; - memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createReq.replica = numOfReplicas + 1; + createReq.replicas[numOfReplicas].id = pDnode->id; + createReq.replicas[numOfReplicas].port = pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createEpset.inUse = 0; createEpset.numOfEps = 1; createEpset.eps[0].port = pDnode->port; memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); - - STransAction action = { - .epSet = createEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_CREATE_MNODE, - .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } + if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; return 0; } @@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); + if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -459,107 +476,28 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO } static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; - SDDropMnodeReq dropReq = {0}; - SSetStandbyReq standbyReq = {0}; - SEpSet alterEpset = {0}; - SEpSet dropEpSet = {0}; - - while (1) { - SMnodeObj *pMObj = NULL; - pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); - if (pIter == NULL) break; - if (pMObj->id == pObj->id) { - sdbRelease(pSdb, pMObj); - continue; - } - - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } - - numOfReplicas++; - sdbRelease(pSdb, pMObj); - } - - alterReq.replica = numOfReplicas; - alterEpset.numOfEps = numOfReplicas; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + int32_t numOfReplicas = 0; + SDDropMnodeReq dropReq = {0}; + SEpSet dropEpSet = {0}; dropReq.dnodeId = pDnode->id; dropEpSet.numOfEps = 1; dropEpSet.eps[0].port = pDnode->port; memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - standbyReq.dnodeId = pDnode->id; - standbyReq.standby = 1; - - { - int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq); - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(MNODE_HANDLE); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_SYNC_SET_MNODE_STANDBY, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_DROP_MNODE, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); + if (totalMnodes == 2) { + mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + } else if (totalMnodes == 3) { + mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes); + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + } else { + return -1; } return 0; @@ -567,7 +505,6 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { if (pObj == NULL) return 0; - if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; if (mndTransAppendNullLog(pTrans) != 0) return -1; @@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { - pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true); if (pShow->pIter == NULL) break; cols = 0; @@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { } static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { +#if 1 + return 0; +#else SMnode *pMnode = pReq->info.node; SDAlterMnodeReq alterReq = {0}; @@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { return -1; } + SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1}; + memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { + mInfo("alter mnode not processed since selfIndex is -1", terrstr()); + return 0; + } + + if (mndWriteFile(pMnode->path, &option) != 0) { + mError("failed to write mnode file since %s", terrstr()); + return -1; + } + SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1}; for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodePort = alterReq.replicas[i].port; - if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i; + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + cfg.myIndex = i; + } } if (cfg.myIndex == -1) { mError("failed to alter mnode since myindex is -1"); return -1; } else { - mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex); + mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); } } - mInfo("trans:-1, sync reconfig will be proposed"); - - SSyncMgmt *pMgmt = &pMnode->syncMgmt; - pMgmt->standby = 0; - int32_t code = syncReconfig(pMgmt->sync, &cfg); + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); if (code != 0) { - mError("trans:-1, failed to propose sync reconfig since %s", terrstr()); - return code; + mError("failed to sync reconfig since %s", terrstr()); } else { - pMgmt->errCode = 0; - taosWLockLatch(&pMgmt->lock); - pMgmt->transId = -1; - taosWUnLockLatch(&pMgmt->lock); - tsem_wait(&pMgmt->syncSem); - mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode)); - terrno = pMgmt->errCode; - return pMgmt->errCode; + mInfo("alter mnode sync success"); + } + + return code; +#endif +} + +static void mndReloadSyncConfig(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + SMnodeObj *pObj = NULL; + ESdbStatus objStatus = 0; + void *pIter = NULL; + bool hasUpdatingMnode = false; + SSyncCfg cfg = {.myIndex = -1}; + + while (1) { + pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false); + if (pIter == NULL) break; + if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) { + mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); + hasUpdatingMnode = true; + } + + if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { + SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; + tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pObj->pDnode->port; + if (pObj->pDnode->id == pMnode->selfDnodeId) { + cfg.myIndex = cfg.replicaNum; + } + cfg.replicaNum++; + } + + sdbReleaseLock(pSdb, pObj, false); + } + + if (cfg.myIndex == -1) { + mInfo("vgId:1, mnode not reload since selfIndex is -1"); + return; + } + + if (!mndGetRestored(pMnode)) { + mInfo("vgId:1, mnode not reload since restore not finished"); + return; + } + + if (hasUpdatingMnode) { + mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); + for (int32_t i = 0; i < cfg.replicaNum; ++i) { + SNodeInfo *pNode = &cfg.nodeInfo[i]; + mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + } + + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); + if (code != 0) { + mError("vgId:1, failed to reconfig mnode sync since %s", terrstr()); + } else { + mInfo("vgId:1, reconfig mnode sync success"); + } } } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0e4dd9fc3c..866a3fa8b9 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { if (!pMnode->deploy) { mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); mndTransPullup(pMnode); - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } else { mInfo("vgId:1, sync restore finished"); } @@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) { snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); syncInfo.pWal = pMnode->pWal; syncInfo.pFsm = mndSyncMakeFsm(pMnode); - syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT; - mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby); - if (pMgmt->standby || pMgmt->replica.id > 0) { - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = 1; - pCfg->myIndex = 0; - SNodeInfo *pNode = &pCfg->nodeInfo[0]; - tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); - pNode->nodePort = pMgmt->replica.port; - mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort); + mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas); + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = pMgmt->numOfReplicas; + pCfg->myIndex = pMgmt->selfIndex; + for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pMgmt->replicas[i].port; + mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); @@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) { setPingTimerMS(pMgmt->sync, 5000); setElectTimerMS(pMgmt->sync, 3000); setHeartbeatTimerMS(pMgmt->sync, 500); - /* - setElectTimerMS(pMgmt->sync, 600); - setHeartbeatTimerMS(pMgmt->sync, 300); - */ mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; @@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); syncStart(pMgmt->sync); - mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); + mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync); } void mndSyncStop(SMnode *pMnode) { @@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) { taosWUnLockLatch(&pMnode->syncMgmt.lock); } -bool mndIsMaster(SMnode *pMnode) { +bool mndIsLeader(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; if (!syncIsReady(pMgmt->sync)) { @@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) { return false; } - if (!pMnode->restored) { + if (!mndGetRestored(pMnode)) { terrno = TSDB_CODE_APP_NOT_READY; return false; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d30cc8f9ca..a64b07fdbd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -53,7 +53,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); -static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } +static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index f922baf329..be93bd2f59 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey); * @param pObj The object of the row. */ void sdbRelease(SSdb *pSdb, void *pObj); +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock); /** * @brief Traverse a sdb table @@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status); +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock); /** * @brief Cancel a traversal diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index ecdf8c71a7..0fee0fbca1 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { taosThreadRwlockUnlock(pLock); } -void sdbRelease(SSdb *pSdb, void *pObj) { +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { if (pObj == NULL) return; SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); if (pRow->type >= SDB_MAX) return; TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + if (lock) { + taosThreadRwlockWrlock(pLock); + } int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); @@ -342,9 +344,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) { sdbFreeRow(pSdb, pRow, true); } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } } +void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); } + void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { *ppObj = NULL; @@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { return ppRow; } -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) { +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) { *ppObj = NULL; SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + if (lock) { + taosThreadRwlockRdlock(pLock); + } SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { @@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat *status = pRow->status; break; } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } return ppRow; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8562230b6a..866561cfc8 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -754,6 +754,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; + ASSERT(pAlterTbReq->colName); if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 59a9f32642..e1578b37f1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -171,11 +171,11 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); - } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); - } + /*if (pRsp->blockNum > 0) {*/ + /*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/ + /*} else {*/ + ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + /*}*/ } int32_t len = 0; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index bcdac1941a..4083a1a0ae 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -245,7 +245,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tqDebug("vgId:%d cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey); continue; } - if (pPushEntry->dataRsp.reqOffset.version > ver) { + if (pPushEntry->dataRsp.reqOffset.version >= ver) { tqDebug("vgId:%d push entry req version %ld, while push version %ld, skip", pTq->pVnode->config.vgId, pPushEntry->dataRsp.reqOffset.version, ver); continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 50c7a7cc53..0ee2e31097 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD if (code) goto _err; } +_err: if (aDelData) { taosArrayDestroy(aDelData); } -_err: return code; } @@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); - if (code) goto _err; *pDelIdx = *pIdx; +_err: if (pDelIdxArray) { taosArrayDestroy(pDelIdxArray); } -_err: return code; } @@ -599,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (code) goto _err; - if (!state->pBlockIdx) { goto _next_fileset; } @@ -898,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs if (code) goto _err; code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } tsdbDelFReaderClose(&pDelFReader); } else { @@ -1231,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { _err: nextRowIterClose(&iter); taosMemoryFreeClear(pTSchema); + *ppLastArray = NULL; + taosArrayDestroy(pColArray); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 995d7a3155..aca9042261 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -396,12 +396,19 @@ _exit: return code; } +static int32_t tDataIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + SDataIter *pIter1 = (SDataIter *)((uint8_t *)n1 - offsetof(SDataIter, n)); + SDataIter *pIter2 = (SDataIter *)((uint8_t *)n2 - offsetof(SDataIter, n)); + + return tRowInfoCmprFn(&pIter1->r, &pIter2->r); +} + static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; int32_t lino = 0; pCommitter->pIter = NULL; - tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pCommitter->rbt, tDataIterCmprFn); // memory TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 85514ed5b6..d08b743763 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -610,9 +610,6 @@ int32_t tsdbFSRollback(STsdbFS *pFS) { ASSERT(0); return code; - -_err: - return code; } int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) { @@ -866,7 +863,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); if (nRef == 0) { tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pSmaF); } } else { @@ -877,7 +874,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { // stt if (sameDisk) { if (pSetNew->nSttF > pSetOld->nSttF) { - ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + ASSERT(pSetNew->nSttF == pSetOld->nSttF + 1); pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -964,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSetOld->pHeadF); } @@ -1104,7 +1101,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbDelFileName(pTsdb, pFS->pDelFile, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pFS->pDelFile); } } @@ -1117,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSet->pHeadF); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index a6628463f8..450032d4f2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -559,6 +559,8 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i // backward put first data row.pTSRow = tGetSubmitBlkNext(&blkIter); + if (row.pTSRow == NULL) return code; + key.ts = row.pTSRow->ts; nRow++; tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 0b7d1cbaf4..a4c5c60e78 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -133,8 +133,15 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { id.uid = pIter->uid; } - tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } double el = (taosGetTimestampUs() - st) / 1000.0; pInfo->elapsedTime += el; @@ -504,9 +511,9 @@ _exit: SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } // SMergeTree ================================================= -static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { - SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode)); - SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) { + SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node)); + SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node)); TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); @@ -543,6 +550,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; + ASSERT(pMTree->pLoadInfo != NULL); for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter *pIter = NULL; @@ -583,7 +591,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node)); + int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index e4080ccf1e..5197823490 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -87,9 +87,13 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { - taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + taosThreadRwlockWrlock(&(*pTsdb)->rwLock); tsdbMemTableDestroy((*pTsdb)->mem); (*pTsdb)->mem = NULL; + taosThreadRwlockUnlock(&(*pTsdb)->rwLock); + + taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + tsdbFSClose(*pTsdb); tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a7850f3114..c346eeef1d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2676,9 +2676,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret int8_t level = 0; int8_t precision = pVnode->config.tsdbCfg.precision; int64_t now = taosGetTimestamp(precision); - int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1 - : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000 - : 1000000); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L + : 1000000L); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; @@ -3372,14 +3372,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl if (pCond->suid != 0) { pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1, - pReader->idStr); + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr); } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr); + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); } } @@ -3395,19 +3394,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - code = doOpenReaderImpl(pReader); + if (numOfTables > 0) { + code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { - return code; + goto _err; } - } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - STsdbReader* pNextReader = pReader->innerReader[1]; + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; // we need only one row pPrevReader->capacity = 1; @@ -3422,19 +3422,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl pNextReader->pMemSchema = pReader->pMemSchema; pNextReader->pReadSnap = pReader->pReadSnap; - code = doOpenReaderImpl(pPrevReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pNextReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pNextReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } @@ -3513,6 +3514,10 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pLReader); } + if (pReader->innerReader[0] != 0) { + tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr); + } + tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 " SMA-time:%.2f ms, fileBlocks:%" PRId64 diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 167b4a104c..84b3afe72c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -18,7 +18,7 @@ // =============== PAGE-WISE FILE =============== static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; - STsdbFD *pFD; + STsdbFD *pFD = NULL; *ppFD = NULL; @@ -35,6 +35,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pFD = taosOpenFile(path, flag); if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD); goto _exit; } pFD->szPage = szPage; @@ -42,11 +43,15 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFD->pFD); taosMemoryFree(pFD); goto _exit; } if (taosStatFile(path, &pFD->szFile, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); goto _exit; } ASSERT(pFD->szFile % szPage == 0); @@ -59,10 +64,12 @@ _exit: static void tsdbCloseFile(STsdbFD **ppFD) { STsdbFD *pFD = *ppFD; - taosMemoryFree(pFD->pBuf); - taosCloseFile(&pFD->pFD); - taosMemoryFree(pFD); - *ppFD = NULL; + if (pFD) { + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); + *ppFD = NULL; + } } static int32_t tsdbWriteFilePage(STsdbFD *pFD) { @@ -117,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { } // check - if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } @@ -443,7 +450,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p pBlockIdx->size = size; pHeadFile->size += size; - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 " size:%" PRId64 " nItem:%d", TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); @@ -457,7 +464,7 @@ _err: int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { int32_t code = 0; SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; - int64_t size; + int64_t size = 0; int64_t n; // check @@ -836,7 +843,8 @@ _err: // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; - SDataFReader *pReader; + int32_t lino = 0; + SDataFReader *pReader = NULL; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fname[TSDB_FILENAME_LEN]; @@ -844,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->pSet = pSet; @@ -852,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // head tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // data tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // sma tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // stt for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppReader = pReader; - return code; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); -_err: - tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; + if (pReader) { + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]); + tsdbCloseFile(&pReader->pSmaFD); + tsdbCloseFile(&pReader->pDataFD); + tsdbCloseFile(&pReader->pHeadFD); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + } return code; } @@ -906,10 +923,6 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { taosMemoryFree(*ppReader); *ppReader = NULL; return code; - -_err: - tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code)); - return code; } int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { @@ -1289,16 +1302,17 @@ _exit: // SDelFWriter ==================================================== int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter; + SDelFWriter *pDelFWriter = NULL; int64_t n; // alloc pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); if (pDelFWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pDelFWriter->pTsdb = pTsdb; pDelFWriter->fDel = *pFile; @@ -1306,21 +1320,28 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // update header code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pDelFWriter->fDel.size = TSDB_FHDR_SIZE; pDelFWriter->fDel.offset = 0; *ppWriter = pDelFWriter; - return code; -_err: - tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; +_exit: + if (code) { + if (pDelFWriter) { + taosMemoryFree(pDelFWriter); + tsdbCloseFile(&pDelFWriter->pWriteH); + } + *ppWriter = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); + } else { + *ppWriter = pDelFWriter; + } return code; } @@ -1456,15 +1477,15 @@ struct SDelFReader { int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; - SDelFReader *pDelFReader; - int64_t n; + SDelFReader *pDelFReader = NULL; // alloc pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); if (pDelFReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } // open impl @@ -1473,14 +1494,18 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH); - if (code) goto _err; + if (code) { + taosMemoryFree(pDelFReader); + goto _exit; + } - *ppReader = pDelFReader; - return code; - -_err: - tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + *ppReader = pDelFReader; + } return code; } @@ -1573,4 +1598,4 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a928dc3484..74c704598d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -67,6 +67,13 @@ extern int32_t tRowInfoCmprFn(const void* p1, const void* p2); extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData); extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); +static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) { + SFDataIter* pIter1 = (SFDataIter*)(((uint8_t*)pNode1) - offsetof(SFDataIter, n)); + SFDataIter* pIter2 = (SFDataIter*)(((uint8_t*)pNode2) - offsetof(SFDataIter, n)); + + return tRowInfoCmprFn(&pIter1->rInfo, &pIter2->rInfo); +} + static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { int32_t code = 0; @@ -79,7 +86,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (code) goto _err; pReader->pIter = NULL; - tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn); // .data file SFDataIter* pIter = &pReader->aFDataIter[0]; @@ -421,7 +428,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { n += tPutDelData((*ppData) + n, pDelData); } - tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%d" PRId64 " size:%d", + tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%" PRId64 " size:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pDelIdx->suid, pDelIdx->uid, size); break; @@ -431,20 +438,21 @@ _exit: return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->pVnode, + tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); return code; } int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { int32_t code = 0; + int32_t lino = 0; STsdbSnapReader* pReader = NULL; // alloc pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->sver = sver; @@ -454,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = taosThreadRwlockRdlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFSRef(pTsdb, &pReader->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = taosThreadRwlockUnlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // data @@ -478,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pIter->aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } else { pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tBlockDataCreate(&pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pReader->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // del pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); if (pReader->aDelIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pReader->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); - *ppReader = pReader; - return code; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code), + pTsdb->path); + *ppReader = NULL; -_err: - tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppReader = NULL; + if (pReader) { + taosArrayDestroy(pReader->aDelData); + taosArrayDestroy(pReader->aDelIdx); + tBlockDataDestroy(&pReader->bData, 1); + tsdbFSDestroy(&pReader->fs); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); + } return code; } @@ -1247,20 +1264,21 @@ _err: // APIs int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { int32_t code = 0; + int32_t lino = 0; STsdbSnapWriter* pWriter = NULL; // alloc pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->pTsdb = pTsdb; pWriter->sver = sver; pWriter->ever = ever; code = tsdbFSCopy(pTsdb, &pWriter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // config pWriter->minutes = pTsdb->keepCfg.days; @@ -1272,7 +1290,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr // SNAP_DATA_TSDB code = tBlockDataCreate(&pWriter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pWriter->fid = INT32_MIN; pWriter->id = (TABLEID){0}; @@ -1280,53 +1298,67 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dReader.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // Writer pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pWriter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dWriter.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataCreate(&pWriter->dWriter.sData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // SNAP_DATA_DEL pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pWriter->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppWriter = pWriter; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + *ppWriter = NULL; - tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppWriter = NULL; + if (pWriter) { + if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW); + if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData); + if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR); + tBlockDataDestroy(&pWriter->dWriter.sData, 1); + tBlockDataDestroy(&pWriter->dWriter.bData, 1); + if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk); + if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx); + tBlockDataDestroy(&pWriter->dReader.bData, 1); + if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx); + tBlockDataDestroy(&pWriter->bData, 1); + tsdbFSDestroy(&pWriter->fs); + taosMemoryFree(pWriter); + } + } else { + tsdbDebug("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); + *ppWriter = pWriter; + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 0a9fbf92a4..f38802aee7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -34,7 +34,9 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - tInitSubmitMsgIter(pMsg, &msgIter); + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return -1; + } while (true) { SSubmitBlkRsp r = {0}; tGetSubmitMsgNext(&msgIter, &pBlock); diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 3c0a041197..730bd264a7 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -141,17 +141,17 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { } void vnodeBufPoolFree(SVBufPool *pPool, void *p) { - uint8_t *ptr = (uint8_t *)p; - SVBufPoolNode *pNode; + // uint8_t *ptr = (uint8_t *)p; + // SVBufPoolNode *pNode; - if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { - pNode = &((SVBufPoolNode *)p)[-1]; - *pNode->pnext = pNode->prev; - pNode->prev->pnext = pNode->pnext; + // if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { + // pNode = &((SVBufPoolNode *)p)[-1]; + // *pNode->pnext = pNode->prev; + // pNode->prev->pnext = pNode->pnext; - pPool->size = pPool->size - sizeof(*pNode) - pNode->size; - taosMemoryFree(pNode); - } + // pPool->size = pPool->size - sizeof(*pNode) - pNode->size; + // taosMemoryFree(pNode); + // } } void vnodeBufPoolRef(SVBufPool *pPool) { diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 314c0e7390..9fe37505ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -46,11 +46,17 @@ int vnodeInit(int nthreads) { return 0; } - vnodeGlobal.stop = 0; + taosThreadMutexInit(&vnodeGlobal.mutex, NULL); + taosThreadCondInit(&vnodeGlobal.hasTask, NULL); + taosThreadMutexLock(&vnodeGlobal.mutex); + + vnodeGlobal.stop = 0; vnodeGlobal.queue.next = &vnodeGlobal.queue; vnodeGlobal.queue.prev = &vnodeGlobal.queue; + taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + vnodeGlobal.nthreads = nthreads; vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); if (vnodeGlobal.threads == NULL) { @@ -59,9 +65,6 @@ int vnodeInit(int nthreads) { return -1; } - taosThreadMutexInit(&vnodeGlobal.mutex, NULL); - taosThreadCondInit(&vnodeGlobal.hasTask, NULL); - for (int i = 0; i < nthreads; i++) { taosThreadCreate(&(vnodeGlobal.threads[i]), NULL, loop, NULL); } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index de3af7cde6..298653d3ed 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -38,7 +38,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { if (taosMkDir(path)) { return TAOS_SYSTEM_ERROR(errno); } - strcpy(dir, path); + snprintf(dir, TSDB_FILENAME_LEN, "%s", path); } if (pCfg) { @@ -51,7 +51,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.commitID = 0; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { - vError("vgId:%d, failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 08c3a34699..9cbb1b8af4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -166,7 +166,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (*ppData) { goto _exit; } else { - pReader->tqHandleDone = 1; + pReader->tqOffsetDone = 1; code = tqOffsetReaderClose(&pReader->pTqOffsetReader); if (code) goto _err; } @@ -219,7 +219,7 @@ _exit: return code; _err: - vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); return code; } @@ -260,7 +260,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr // commit it code = vnodeCommit(pVnode); - if (code) goto _err; + if (code) { + taosMemoryFree(pWriter); + goto _err; + } // inc commit ID pVnode->state.commitID++; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 131327976f..b531cf7574 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -43,14 +43,23 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); tStartDecode(&dc); - tDecodeI32v(&dc, &nReqs); + if (tDecodeI32v(&dc, &nReqs) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } for (int32_t iReq = 0; iReq < nReqs; iReq++) { tb_uid_t uid = tGenIdPI64(); char *name = NULL; tStartDecode(&dc); - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; @@ -68,7 +77,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int64_t ctime = taosGetTimestampMs(); tb_uid_t uid; - tInitSubmitMsgIter(pSubmitReq, &msgIter); + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + code = terrno; + goto _err; + } for (;;) { tGetSubmitMsgNext(&msgIter, &pBlock); @@ -78,10 +90,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { char *name = NULL; tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } uid = metaGetTableEntryUidByName(pVnode->pMeta, name); if (uid == 0) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f9e47f92fe..fc996a6003 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -212,6 +212,7 @@ typedef struct SExprSupp { int32_t numOfExprs; // the number of scalar expression in group operator SqlFunctionCtx* pCtx; int32_t* rowEntryInfoOffset; // offset value for each row result cell info + SFilterInfo* pFilterInfo; } SExprSupp; typedef struct SOperatorInfo { @@ -479,6 +480,7 @@ typedef struct SStreamScanInfo { SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; int32_t primaryTsIndex; // primary time stamp slot id SReadHandle readHandle; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. @@ -775,6 +777,7 @@ typedef struct SStreamPartitionOperatorInfo { SPartitionBySupporter partitionSup; SExprSupp scalarSup; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; SHashObj* pPartitions; void* parIte; SSDataBlock* pInputDataBlock; @@ -924,7 +927,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6a43ea40c6..ce0ff485ab 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); tFreeSSubmitRsp(pInserter->submitRes.pRsp); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index df5de20d10..994be55996 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1191,14 +1191,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; + tExprNode* pExprNode = pExp->pExpr; + + pExprNode->_function.functionId = pFuncNode->funcId; + pExprNode->_function.pFunctNode = pFuncNode; + + tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); #if 1 // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + const char* name = "tbname"; + int32_t len = strlen(name); + + if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) && + pExprNode->_function.functionName[len] == 0) { + pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e527f7e7c1..413813f58a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; - qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts); + qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid); + qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); return -1; } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid); + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; tsdbReaderClose(pInfo->dataReader); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8c1104f519..e59755787a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1113,15 +1113,24 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo) { if (pFilterNode == NULL || pBlock->info.rows == 0) { return; } - SFilterInfo* filter = NULL; + SFilterInfo* filter = pFilterInfo; + int64_t st = taosGetTimestampUs(); + +// pError("start filter"); // todo move to the initialization function - int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + int32_t code = 0; + bool needFree = false; + if (filter == NULL) { + needFree = true; + code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + } + SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); @@ -1130,7 +1139,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM // todo the keep seems never to be True?? bool keep = filterExecute(filter, pBlock, &p, NULL, param1.numOfCols, &status); - filterFreeInfo(filter); + + if (needFree) { + filterFreeInfo(filter); + } extractQualifiedTupleByFilterResult(pBlock, p, keep, status); @@ -1845,6 +1857,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); return pTaskInfo->code; } @@ -2479,7 +2492,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pAggInfo->pCondition, pInfo->pRes, NULL); + doFilter(pAggInfo->pCondition, pInfo->pRes, NULL, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -2873,7 +2886,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { break; } - doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo, NULL); if (fillResult->info.rows > 0) { break; } @@ -3049,6 +3062,12 @@ void cleanupExprSupp(SExprSupp* pSupp) { destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); taosMemoryFreeClear(pSupp->pExprInfo); } + + if (pSupp->pFilterInfo != NULL) { + filterFreeInfo(pSupp->pFilterInfo); + pSupp->pFilterInfo = NULL; + } + taosMemoryFree(pSupp->rowEntryInfoOffset); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f3ae652987..e62f72c8f8 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -311,7 +311,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes, NULL); + doFilter(pInfo->pCondition, pRes, NULL, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -991,6 +991,8 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFree(pInfo->partitionSup.keyBuf); cleanupExprSupp(&pInfo->scalarSup); + cleanupExprSupp(&pInfo->tbnameCalSup); + cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); taosMemoryFreeClear(param); } @@ -1037,6 +1039,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + if (pPartNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + int32_t keyLen = 0; code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index b4aa6de0a8..53cfa6c27a 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -387,7 +387,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { break; } if (pJoinInfo->pCondAfterMerge != NULL) { - doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL); + doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL, NULL); } if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index e9e6fed66a..fe6ee4129a 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -315,7 +315,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } // do apply filter - doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL); + doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL, NULL); // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { @@ -325,7 +325,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } else { // do apply filter if (pRes->info.rows > 0) { - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); if (pRes->info.rows == 0) { continue; } @@ -518,7 +518,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { } } - doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); + doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL, NULL); size_t rows = pInfo->pRes->info.rows; if (rows > 0 || pOperator->status == OP_EXEC_DONE) { break; @@ -620,7 +620,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { } pRes->info.rows = 1; - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); /*int32_t status = */ doIngroupLimitOffset(&pProjectInfo->limitInfo, 0, pRes, pOperator); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 950e4b220a..856129b231 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -421,7 +421,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampUs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, pOperator->exprSupp.pFilterInfo); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -790,6 +790,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; + + if (pInfo->pFilterNode != NULL) { + code = filterInitFromNode((SNode*)pInfo->pFilterNode, &pOperator->exprSupp.pFilterInfo, 0); + } + pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; pInfo->currentGroupId = -1; @@ -1159,7 +1164,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 return NULL; } - doFilter(pInfo->pCondition, pResult, NULL); + doFilter(pInfo->pCondition, pResult, NULL, NULL); if (pResult->info.rows == 0) { continue; } @@ -1347,10 +1352,15 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL); + projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, pTagCalSup->numOfExprs, NULL); ASSERT(pResBlock->info.rows == 1); // build tagArray + /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/ + /*STagVal tagVal = {*/ + /*.cid = 0,*/ + /*.type = 0,*/ + /*};*/ // build STag // set STag @@ -1505,7 +1515,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } if (filter) { - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); } blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); @@ -1927,7 +1937,7 @@ FETCH_NEXT_BLOCK: } } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { @@ -2146,6 +2156,9 @@ static void destroyStreamScanOperatorInfo(void* param) { taosMemoryFree(pStreamScan->pPseudoExpr); } + cleanupExprSupp(&pStreamScan->tbnameCalSup); + cleanupExprSupp(&pStreamScan->tagCalSup); + updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); @@ -2200,6 +2213,19 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } + if (pTableScanNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2397,7 +2423,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } @@ -2578,16 +2604,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SMetaReader smrChildTable = {0}; metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smrChildTable, condTableName); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { metaReaderClear(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } + SMetaReader smrSuperTable = {0}; metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); metaReaderClear(&smrSuperTable); metaReaderClear(&smrChildTable); @@ -3577,7 +3614,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, NULL); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -3912,6 +3949,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pColList); goto _error; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index ed3247e10e..0d5f521034 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { return NULL; } - doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo); + doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo, NULL); if (blockDataGetNumOfRows(pBlock) == 0) { continue; } @@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 5be9918185..8b716531e5 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1501,7 +1501,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } doStreamFillImpl(pOperator); - doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo, NULL); pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; if (pInfo->pRes->info.rows > 0) { break; @@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); pInfo->pCondition = pPhyFillNode->node.pConditions; pInfo->pColMatchColInfo = pColMatchColInfo; - initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pInfo->srcRowIndex = 0; pOperator->name = "FillOperator"; @@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi _error: destroyStreamFillOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index df16914cb8..9c219d2765 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1277,7 +1277,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -1315,7 +1315,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBlock, NULL); + doFilter(pInfo->pCondition, pBlock, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; + STimeWindow win = {0}; if (IS_FINAL_OP(pInfo)) { win.skey = startTsCols[i]; @@ -2019,7 +2020,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -2062,7 +2063,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -5251,7 +5252,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes); - doFilter(pMiaInfo->pCondition, pRes, NULL); + doFilter(pMiaInfo->pCondition, pRes, NULL, NULL); if (pRes->info.rows >= pOperator->resultInfo.capacity) { break; } @@ -5828,27 +5829,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->isFinal = false; - if (pIntervalPhyNode->window.pExprs != NULL) { - int32_t numOfScalar = 0; - SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; + + initBasicInfo(&pInfo->binfo, pResBlock); + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); pInfo->invertible = false; diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index cdc4e66e42..9e89955ae5 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -478,7 +478,10 @@ static int32_t tlvDecodeValueEnum(STlvDecoder* pDecoder, void* pValue, int16_t l return code; } -static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) { +static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue, int32_t size) { + if (pTlv->len > size - 1) { + return TSDB_CODE_FAILED; + } memcpy(pValue, pTlv->value, pTlv->len); return TSDB_CODE_SUCCESS; } @@ -919,9 +922,14 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) { } break; } - case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_JSON: { + if (pTlv->len <= 0 || pTlv->len > TSDB_MAX_JSON_TAG_LEN) { + code = TSDB_CODE_FAILED; + break; + } code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p); break; + } case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: // todo @@ -1097,7 +1105,7 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); break; case FUNCTION_CODE_FUNCTION_NAME: - code = tlvDecodeCStr(pTlv, pNode->functionName); + code = tlvDecodeCStr(pTlv, pNode->functionName, sizeof(pNode->functionName)); break; case FUNCTION_CODE_FUNCTION_ID: code = tlvDecodeI32(pTlv, &pNode->funcId); @@ -1226,10 +1234,10 @@ static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI32(pTlv, &pNode->acctId); break; case NAME_CODE_DB_NAME: - code = tlvDecodeCStr(pTlv, pNode->dbname); + code = tlvDecodeCStr(pTlv, pNode->dbname, sizeof(pNode->dbname)); break; case NAME_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tname); + code = tlvDecodeCStr(pTlv, pNode->tname, sizeof(pNode->tname)); break; default: break; @@ -1538,7 +1546,7 @@ static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) { tlvForEach(pDecoder, pTlv, code) { switch (pTlv->type) { case EP_CODE_FQDN: - code = tlvDecodeCStr(pTlv, pNode->fqdn); + code = tlvDecodeCStr(pTlv, pNode->fqdn, sizeof(pNode->fqdn)); break; case EP_CODE_port: code = tlvDecodeU16(pTlv, &pNode->port); @@ -3207,7 +3215,7 @@ static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_QUERY_INSERT_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tableName); + code = tlvDecodeCStr(pTlv, pNode->tableName, sizeof(pNode->tableName)); break; case PHY_QUERY_INSERT_CODE_VG_ID: code = tlvDecodeI32(pTlv, &pNode->vgId); @@ -3284,10 +3292,10 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_DELETER_CODE_TABLE_FNAME: - code = tlvDecodeCStr(pTlv, pNode->tableFName); + code = tlvDecodeCStr(pTlv, pNode->tableFName, sizeof(pNode->tableFName)); break; case PHY_DELETER_CODE_TS_COL_NAME: - code = tlvDecodeCStr(pTlv, pNode->tsColName); + code = tlvDecodeCStr(pTlv, pNode->tsColName, sizeof(pNode->tsColName)); break; case PHY_DELETER_CODE_DELETE_TIME_RANGE: code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 9196f77124..bc0c4d42bb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -209,6 +209,10 @@ int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) { } SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + if (NULL == pAllocator) { + nodesError("allocator id %" PRIx64 " weak reference failed", allocatorId); + return -1; + } return pAllocator->self; } @@ -1716,9 +1720,10 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = 0; if ('\0' == pCol->tableAlias[0]) { - len = sprintf(name, "%s", pCol->colName); + len = snprintf(name, sizeof(name), "%s", pCol->colName); + } else { + len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } - len = sprintf(name, "%s.%s", pCol->tableAlias, pCol->colName); if (NULL == taosHashGet(pCxt->pColHash, name, len)) { pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); if (TSDB_CODE_SUCCESS == pCxt->errCode) { diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index 787abf287e..fb4b46aa35 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -175,8 +175,6 @@ _end: void taosCleanupKeywordsTable(); -SToken tscReplaceStrToken(char **str, SToken *token, const char *newToken); - SToken taosTokenDup(SToken *pToken, char *buf, int32_t len); #ifdef __cplusplus diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2e6f839368..15b939f14b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -518,7 +518,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa if (NULL != pDbName) { COPY_STRING_FORM_ID_TOKEN(realTable->table.dbName, pDbName); } else { - strcpy(realTable->table.dbName, pCxt->pQueryCxt->db); + snprintf(realTable->table.dbName, sizeof(realTable->table.dbName), "%s", pCxt->pQueryCxt->db); } if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) { COPY_STRING_FORM_ID_TOKEN(realTable->table.tableAlias, pTableAlias); @@ -599,7 +599,7 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { state->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == state->pCol) { nodesDestroyNode((SNode*)state); - CHECK_OUT_OF_MEM(state->pCol); + CHECK_OUT_OF_MEM(NULL); } state->pExpr = pExpr; return (SNode*)state; @@ -613,7 +613,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode interval->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == interval->pCol) { nodesDestroyNode((SNode*)interval); - CHECK_OUT_OF_MEM(interval->pCol); + CHECK_OUT_OF_MEM(NULL); } interval->pInterval = pInterval; interval->pOffset = pOffset; @@ -631,7 +631,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == fill->pWStartTs) { nodesDestroyNode((SNode*)fill); - CHECK_OUT_OF_MEM(fill->pWStartTs); + CHECK_OUT_OF_MEM(NULL); } strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart"); return (SNode*)fill; @@ -1495,10 +1495,10 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno CHECK_OUT_OF_MEM(pStmt); pStmt->indexType = type; pStmt->ignoreExists = ignoreExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->dbName, sizeof(pStmt->dbName), "%s", ((SRealTableNode*)pRealTable)->table.dbName); + snprintf(pStmt->tableName, sizeof(pStmt->tableName), "%s", ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pIndexName); nodesDestroyNode(pRealTable); pStmt->pCols = pCols; @@ -1524,8 +1524,8 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->ignoreNotExists = ignoreNotExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); nodesDestroyNode(pIndexName); return (SNode*)pStmt; } @@ -1818,7 +1818,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(pFunc); - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) { nodesDestroyNode((SNode*)pFunc); CHECK_OUT_OF_MEM(NULL); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index eec20f91e4..5b861ef79a 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -378,7 +378,9 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time } else if (pToken->type == TK_TODAY) { ts = taosGetTimestampToday(timePrec); } else if (pToken->type == TK_NK_INTEGER) { - toInteger(pToken->z, pToken->n, 10, &ts); + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } } else { // parse the RFC-3339/ISO-8601 timestamp format string if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -591,8 +593,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } return func(pMsgBuf, &iv, pSchema->bytes, param); } @@ -894,10 +894,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } - val->i64 = iv; break; } @@ -1173,7 +1170,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* NEXT_TOKEN(pCxt->pSql, sToken); SName sname; - createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); + CHECK_CODE(createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&sname, dbFName); strcpy(pCxt->sTableName, sname.tname); @@ -1605,7 +1602,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (!pCxt->pComCxt->needMultiParse) { continue; } else { - parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); + parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); break; } } @@ -1613,7 +1610,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } - parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); @@ -1650,7 +1647,7 @@ static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) { pCxt->pComCxt->needMultiParse = false; return TSDB_CODE_SUCCESS; } - parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); // merge according to vgId if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index e8c877bed2..997e680c99 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -// clang-format off + #include "parInsertData.h" #include "catalog.h" @@ -25,8 +25,8 @@ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; + TSKEY skey; + void* payloadAddr; int16_t index; } SBlockKeyTuple; @@ -194,8 +194,8 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { SEncoder coder = {0}; - char* pBuf; - int32_t len; + char* pBuf; + int32_t len; int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); @@ -211,19 +211,19 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) } } - pBuf= pBlocks->pData + pBlocks->size; + pBuf = pBlocks->pData + pBlocks->size; tEncoderInit(&coder, pBuf, len); - tEncodeSVCreateTbReq(&coder, pCreateTbReq); + int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); tEncoderClear(&coder); - pBlocks->size += len; pBlocks->createTbReqLen = len; - return TSDB_CODE_SUCCESS; + + return code; } -int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, +int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, + int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); @@ -272,12 +272,12 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); -// if (!pDataBlock->cloned) { - // free the refcount for metermeta - taosMemoryFreeClear(pDataBlock->pTableMeta); + // if (!pDataBlock->cloned) { + // free the refcount for metermeta + taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); -// } + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + // } taosMemoryFreeClear(pDataBlock); } @@ -687,16 +687,16 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p STableDataBlocks** p = taosHashIterate(pHashObj, NULL); STableDataBlocks* pOneTableBlock = *p; SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger *pBlkRowMerger = NULL; + SBlockRowMerger* pBlkRowMerger = NULL; while (pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = - getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); + pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId + int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), + TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, + pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); @@ -708,7 +708,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p // the maximum expanded size in byte when a row-wise data is converted to SDataRow format int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen; + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + + pOneTableBlock->createTbReqLen; if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); @@ -861,7 +862,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); if (pBlock->pTableMeta) { - void *pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); if (NULL == pNewMeta) { taosMemoryFreeClear(*pDst); return TSDB_CODE_OUT_OF_MEMORY; @@ -887,20 +888,18 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI } pBlock->vgId = vgId; - + if (pBlock->pTableMeta) { pBlock->pTableMeta->uid = uid; pBlock->pTableMeta->vgId = vgId; } - + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); return TSDB_CODE_SUCCESS; } -STableMeta *qGetTableMetaInDataBlock(void* pDataBlock) { - return ((STableDataBlocks*)pDataBlock)->pTableMeta; -} +STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } void qFreeStmtDataBlock(void* pDataBlock) { if (pDataBlock == NULL) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 345b9cc5b9..76c01cfd90 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -613,27 +613,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { return 0; } -SToken tscReplaceStrToken(char** str, SToken* token, const char* newToken) { - char* src = *str; - size_t nsize = strlen(newToken); - int32_t size = (int32_t)strlen(*str) - token->n + (int32_t)nsize + 1; - int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); - SToken ntoken; - - *str = taosMemoryCalloc(1, size); - - strncpy(*str, src, bsize); - strcat(*str, newToken); - strcat(*str, token->z + token->n); - - ntoken.n = (uint32_t)nsize; - ntoken.z = *str + bsize; - - taosMemoryFreeClear(src); - - return ntoken; -} - SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { SToken t0 = {0}; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2cef80c4cb..dbe0c97ab6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -292,9 +292,13 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { } else { do { SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (NULL == pTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (pCxt->currLevel == currTotalLevel) { taosArrayPush(pTables, &pTable); if (hasSameTableAlias(pTables)) { + taosArrayDestroy(pTables); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias); } @@ -308,7 +312,7 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { SFullDatabaseName name = {0}; - strcpy(name.fullDbName, pFullDbName); + snprintf(name.fullDbName, sizeof(name.fullDbName), "%s", pFullDbName); return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName)); } @@ -540,15 +544,17 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray if (TSDB_CODE_SUCCESS == code) { code = collectUseTable(pName, pCxt->pTables); } - if (pParCxt->async) { - code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); - } else { - SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, - .requestId = pParCxt->requestId, - .requestObjRefId = pParCxt->requestRid, - .mgmtEps = pParCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS == code) { + if (pParCxt->async) { + code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); + } else { + SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; - code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + } } if (TSDB_CODE_SUCCESS != code) { parserError("0x%" PRIx64 " getTableIndex error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, @@ -990,9 +996,9 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) return pCxt->errCode; } if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.u; + pVal->datum.i = (int64_t)pVal->datum.u; } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.d; + pVal->datum.i = (int64_t)pVal->datum.d; } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { pVal->datum.i = pVal->datum.b; } @@ -1069,7 +1075,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_BIGINT: { code = toInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.i); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_BIGINT(pVal->datum.i))) { + if (strict && TSDB_CODE_SUCCESS != code) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -3430,10 +3436,10 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return NULL; } pCol->node.resType = ((SExprNode*)pNode)->resType; - strcpy(pCol->tableAlias, pTableAlias); - strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); - strcpy(pCol->node.aliasName, pCol->colName); - strcpy(pCol->node.userAlias, ((SExprNode*)pNode)->userAlias); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pTableAlias); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", ((SExprNode*)pNode)->aliasName); + snprintf(pCol->node.aliasName, sizeof(pCol->node.aliasName), "%s", pCol->colName); + snprintf(pCol->node.userAlias, sizeof(pCol->node.userAlias), "%s", ((SExprNode*)pNode)->userAlias); return (SNode*)pCol; } @@ -3740,8 +3746,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int32_t val, int32_t minVal, int32_t maxVal) { if (val >= 0 && (val < minVal || val > maxVal)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 " valid range: [%d, %d]", pName, - val, minVal, maxVal); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %d valid range: [%d, %d]", pName, val, + minVal, maxVal); } return TSDB_CODE_SUCCESS; } @@ -3878,7 +3884,7 @@ static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pO static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) { if (val >= 0 && val != v1 && val != v2) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option %s: %" PRId64 ", only %d, %d allowed", pName, val, v1, v2); + "Invalid option %s: %d, only %d, %d allowed", pName, val, v1, v2); } return TSDB_CODE_SUCCESS; } @@ -4463,8 +4469,8 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pTable->table.dbName, pInfo->pDbName); - strcpy(pTable->table.tableName, pInfo->pTableName); + snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); + snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); pSelect->pFromTable = (SNode*)pTable; @@ -4940,9 +4946,9 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt alterReq.superUser = 0; alterReq.enable = pStmt->enable; alterReq.sysInfo = pStmt->sysinfo; - strcpy(alterReq.pass, pStmt->password); + snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password); if (NULL != pCxt->pParseCxt->db) { - strcpy(alterReq.dbname, pCxt->pParseCxt->db); + snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db); } return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq); @@ -6049,7 +6055,7 @@ static SNode* createProjectCol(const char* pProjCol) { if (NULL == pCol) { return NULL; } - strcpy(pCol->colName, pProjCol); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pProjCol); return (SNode*)pCol; } @@ -6082,9 +6088,9 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32 nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pRealTable->table.dbName, pDb); - strcpy(pRealTable->table.tableName, pTable); - strcpy(pRealTable->table.tableAlias, pTable); + snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); + snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); + snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); pSelect->pFromTable = (SNode*)pRealTable; if (numOfProjs >= 0) { @@ -7125,8 +7131,9 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p pReq->newComment = strdup(pStmt->pOptions->comment); if (NULL == pReq->newComment) { code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pReq->newCommentLen = strlen(pReq->newComment); } - pReq->newCommentLen = strlen(pReq->newComment); } else { pReq->newCommentLen = -1; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dab1f01574..ee82e6c04b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -381,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); + taosMemoryFree(tmp); goto end; } val.nData = valLen; @@ -652,8 +653,8 @@ static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCac } SUserAuthInfo auth = {0}; - strcpy(auth.user, pCxt->pUser); - strcpy(auth.dbFName, p->dbFName); + snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser); + snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName); auth.type = AUTH_TYPE_WRITE; taosArrayPush(pCatalogReq->pUser, &auth); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e149460550..9e1a01b13d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -471,9 +471,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) return NULL; } pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pExpr->aliasName); if (NULL != pStmtName) { - strcpy(pCol->tableAlias, pStmtName); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName); } return pCol; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 50f1f4b369..cd02e7a136 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1124,7 +1124,7 @@ static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, SNodeList** pS bool notOptimize = false; int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, ¬Optimize, pSequencingNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { - nodesClearList(*pSequencingNodes); + NODES_CLEAR_LIST(*pSequencingNodes); } return code; } @@ -1361,74 +1361,6 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo return code; } -static SNode* smaIndexOptCreateWStartTs() { - SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pWStart) { - return NULL; - } - strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pWStart, NULL, 0)) { - nodesDestroyNode((SNode*)pWStart); - return NULL; - } - return (SNode*)pWStart; -} - -static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { - SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pMergeKey) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->pExpr = nodesCloneNode(pCol); - if (NULL == pMergeKey->pExpr) { - nodesDestroyNode((SNode*)pMergeKey); - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->order = ORDER_ASC; - pMergeKey->nullOrder = NULL_ORDER_FIRST; - return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); -} - -static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { - if (wstrartIndex < 0) { - SNode* pWStart = smaIndexOptCreateWStartTs(); - if (NULL == pWStart) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t code = createColumnByRewriteExpr(pWStart, &pInterval->node.pTargets); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pWStart); - return code; - } - wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; - } - return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); -} - -static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { - SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; - SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); - if (NULL == pMergeTargets) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SLogicNode* pSmaScan = NULL; - SLogicNode* pMerge = NULL; - SNodeList* pMergeKeys = NULL; - int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); - } - return code; -} - static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pSmaCols, int32_t wstrartIndex) { SLogicNode* pSmaScan = NULL; @@ -1559,7 +1491,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) { return NULL; } - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { SColumnNode* pCol = (SColumnNode*)pNode; partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName); @@ -2028,7 +1960,7 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode if (NULL != pSelectValue) { strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); } else { - snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc); + snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc); } int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol)); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 81a3a2cb16..8ff8538c0d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -67,7 +67,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const S if (NULL == pSlot) { return NULL; } - strcpy(pSlot->name, pName); + snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -663,13 +663,17 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); } - SNodeList* condCols = nodesMakeList(); if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { - code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols); - } - if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc); - nodesDestroyList(condCols); + SNodeList* pCondCols = nodesMakeList(); + if (NULL == pCondCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &pCondCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pCondCols, pJoin->node.pOutputDataBlockDesc); + } + nodesDestroyList(pCondCols); } if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { @@ -1634,7 +1638,7 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; if (NULL != pCxt->pPlanCxt->pUser) { - strcpy(pSubplan->user, pCxt->pPlanCxt->pUser); + snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser); } return pSubplan; } @@ -1825,7 +1829,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNode* pSubplan, int32_t lev return TSDB_CODE_OUT_OF_MEMORY; } } - return nodesListStrictAppend(pGroup->pNodeList, (SNode*)pSubplan); + return nodesListAppend(pGroup->pNodeList, (SNode*)pSubplan); } static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 74ed3b57a4..9567a2388f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -342,7 +342,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart); int32_t code = fmGetFuncInfo(pWStart, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pFuncs, (SNode*)pWStart); @@ -367,7 +367,7 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWEnd->functionName, "_wend"); - snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, pWEnd); + snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd); int32_t code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd); @@ -389,22 +389,19 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic SNode* pConditions = pMergeWindow->node.pConditions; pMergeWindow->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SWindowLogicNode* pPartWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow); if (NULL == pPartWin) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartWin->node.groupAction = GROUP_ACTION_KEEP; + pMergeWindow->node.pTargets = pTargets; + pMergeWindow->node.pConditions = pConditions; + pPartWin->node.pChildren = pChildren; + splSetParent((SLogicNode*)pPartWin); - if (TSDB_CODE_SUCCESS == code) { - pMergeWindow->node.pTargets = pTargets; - pMergeWindow->node.pConditions = pConditions; - pPartWin->node.pChildren = pChildren; - splSetParent((SLogicNode*)pPartWin); - code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); - } int32_t index = 0; + int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = stbSplAppendWStart(pPartWin->pFuncs, &index); } @@ -721,15 +718,16 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO SNode* pConditions = pMergeAgg->node.pConditions; pMergeAgg->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SAggLogicNode* pPartAgg = (SAggLogicNode*)nodesCloneNode((SNode*)pMergeAgg); if (NULL == pPartAgg) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartAgg->node.groupAction = GROUP_ACTION_KEEP; - if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 626e78b4d3..36c6817595 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -123,6 +123,7 @@ typedef struct SQWTaskCtx { int32_t execId; int32_t level; + bool queryGotData; bool queryRsped; bool queryEnd; bool queryContinue; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9115c2d3aa..d2c16a1cb5 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -10,6 +10,7 @@ #include "tdatablock.h" #include "tmsg.h" #include "tname.h" +#include "tglobal.h" SQWorkerMgmt gQwMgmt = { .lock = 0, @@ -92,6 +93,19 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { return TSDB_CODE_SUCCESS; } +int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) { + if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) { + if (!ctx->localExec) { + qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode)); + } + + ctx->queryRsped = true; + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; @@ -144,7 +158,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (numOfResBlock == 0) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); } else { - QW_TASK_DLOG("qExecTask done", ""); + QW_TASK_DLOG("qExecTask done, useconds:%" PRIu64, useconds); } dsEndPut(sinkHandle, useconds); @@ -234,6 +248,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t code = 0; SOutputData output = {0}; + if (NULL == ctx->sinkHandle) { + return TSDB_CODE_SUCCESS; + } + *dataLen = 0; while (true) { @@ -407,6 +425,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (!ctx->queryRsped) { QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); @@ -419,6 +442,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); @@ -499,21 +527,17 @@ _return: if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC); + ctx->queryGotData = true; } - if (QW_PHASE_POST_QUERY == phase && ctx) { - if (!ctx->localExec) { - bool rsped = false; - SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; - qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); - qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); - if (!rsped) { - qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); - } + if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) { + bool rsped = false; + SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + if (!rsped) { + qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false); } - - ctx->queryRsped = true; } if (ctx) { @@ -551,6 +575,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); ctx->ctrlConnInfo = qwMsg->connInfo; + ctx->phase = -1; QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); @@ -604,6 +629,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } + qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); @@ -619,6 +646,31 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); + if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + void *rsp = NULL; + int32_t dataLen = 0; + SOutputData sOutput = {0}; + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + + if (rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + } + + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, + tstrerror(code), dataLen); + } + } + QW_RET(TSDB_CODE_SUCCESS); } @@ -740,7 +792,9 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { locked = true; // RC WARNING - if (QW_QUERY_RUNNING(ctx)) { + if (-1 == ctx->phase || false == ctx->queryGotData) { + QW_TASK_DLOG_E("task query unfinished"); + } else if (QW_QUERY_RUNNING(ctx)) { atomic_store_8((int8_t *)&ctx->queryContinue, 1); } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 2678a652a9..33bf50d853 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -913,7 +913,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); +// SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); @@ -993,6 +993,12 @@ int32_t schLaunchTaskImpl(void *param) { SCH_ERR_JRET(schLaunchRemoteTask(pJob, pTask)); } +#if 0 + if (SCH_IS_QUERY_JOB(pJob)) { + SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask)); + } +#endif + _return: if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 813a11c056..c4b45649a4 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -155,8 +155,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { return; } - SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", errCode:0x%x", *jobId, errCode); - + SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); schHandleJobDrop(pJob, errCode); schReleaseJob(*jobId); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 4a63cd3bb2..ee317d0751 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -67,7 +67,11 @@ void streamSchedByTimer(void* param, void* tmrId) { atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - streamTaskInput(pTask, (SStreamQueueItem*)trigger); + if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) { + taosFreeQitem(trigger); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + return; + } streamSchedExec(pTask); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 63a28e17ed..53e49a6ba5 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -28,6 +28,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF sprintf(streamPath, "%s/%s", path, "stream"); pMeta->path = strdup(streamPath); if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db) < 0) { + taosMemoryFree(streamPath); goto _err; } @@ -58,7 +59,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF return pMeta; _err: - if (pMeta->path) taosMemoryFree(pMeta->path); + taosMemoryFree(pMeta->path); if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks); if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb); @@ -250,6 +251,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { + tdbFree(pKey); + tdbFree(pVal); return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); @@ -257,10 +260,14 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { tDecoderClear(&decoder); if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ea590c57e6..c0ed135d74 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -63,7 +63,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int sprintf(statePath, "%s/%d", path, pTask->taskId); } else { memset(statePath, 0, 300); - strncpy(statePath, path, 300); + tstrncpy(statePath, path, 300); } if (tdbOpen(statePath, szPage, pages, &pState->db) < 0) { goto _err; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 7c4aaba6ce..5aadb599a3 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -174,14 +174,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { if (res == TSDB_CODE_SUCCESS) { return false; } else { - qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 - ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } } - qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } @@ -203,14 +198,10 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { } if (ts < pInfo->minTS) { - qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } else if (res == TSDB_CODE_SUCCESS) { return false; } - qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, - maxTs, *pMapMaxTs, ts); // check from tsdb api return true; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0afc373f2d..101e99afea 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -169,7 +169,7 @@ typedef struct SSyncNode { } SSyncNode; // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 378ac743b3..9343b45dc7 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -86,10 +86,10 @@ void syncCleanUp() { } } -int64_t syncOpen(const SSyncInfo* pSyncInfo) { +int64_t syncOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); if (pSyncNode == NULL) { - sError("failed to open sync node. vgId:%d", pSyncInfo->vgId); + sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr()); return -1; } @@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg return ret; } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { } ASSERT(rid == pSyncNode->rid); +#if 0 if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; @@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; +#else + syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); + syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return 0; +#endif } int32_t syncLeaderTransfer(int64_t rid) { @@ -919,9 +926,7 @@ _END: } // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { - SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; - +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode)); if (pSyncNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { sError("failed to open raft cfg file. path:%s", pSyncNode->configPath); goto _error; } - pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + if (pSyncInfo->syncCfg.replicaNum == 0) { + pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + } raftCfgClose(pSyncNode->pRaftCfg); pSyncNode->pRaftCfg = NULL; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a148c1e36b..7afeb31d4c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -987,7 +987,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch a new ofp and make it dirty SPgno pgno = 0; - SPage *ofp, *nextOfp; + SPage *ofp = NULL, *nextOfp = NULL; ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 04711eb6a0..58be8642d7 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -34,9 +34,9 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); -static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) { - SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode)); - SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) { + SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node)); + SPage *pPageR = (SPage *)(((uint8_t *)rhs) - offsetof(SPage, node)); SPgno pgnoL = TDB_PAGE_PGNO(pPageL); SPgno pgnoR = TDB_PAGE_PGNO(pPageR); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 83323725be..359b9824cf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -597,6 +597,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; + tDebug("%s conn %p alloc read buf", CONN_GET_INST_LABEL(conn), conn); transAllocBuffer(pBuf, buf); } static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { @@ -609,7 +610,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; while (transReadComplete(pBuf)) { - tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); + tDebug("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); if (pBuf->invalid) { cliHandleExcept(conn); break; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 42079ff005..f843c8559c 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -203,7 +203,7 @@ bool transReadComplete(SConnBuffer* connBuf) { } int transSetConnOption(uv_tcp_t* stream) { - uv_tcp_nodelay(stream, 1); + uv_tcp_nodelay(stream, 0); int ret = uv_tcp_keepalive(stream, 5, 60); return ret; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 57bf8a34b3..3616b4808f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -332,7 +332,10 @@ void uvOnSendCb(uv_write_t* req, int status) { if (status == 0) { tTrace("conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { - SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + STraceId* trace = &msg->msg.info.traceId; + tGDebug("conn %p write data out", conn); + destroySmsg(msg); // send cached data if (!transQueueEmpty(&conn->srvMsgs)) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e49a963191..144cce2cd0 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -351,7 +351,10 @@ int walCheckAndRepairIdx(SWal* pWal) { taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); break; } - taosLSeekFile(pIdxFile, offset, SEEK_SET); + if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d cannot seek offset %ld when repair idx since %s", pWal->cfg.vgId, offset, terrstr()); + } int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -636,6 +639,5 @@ int walRemoveMeta(SWal* pWal) { if (metaVer == -1) return 0; char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); - taosRemoveFile(fnameStr); - return 0; + return taosRemoveFile(fnameStr); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c4f1a81eeb..b2cd4fac11 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -514,7 +514,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { pReader->pHead->head.version, ver); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -528,7 +527,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 628b432446..e7161079d9 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -42,10 +42,20 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, i); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); } } walRemoveMeta(pWal); diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 65f1bac60a..cbbf53d8f1 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -219,7 +219,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { while (temp != pTree->NIL) { y = temp; - int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp)); + int32_t c = pTree->cmprFn(z, temp); if (c < 0) { temp = temp->left; } else if (c > 0) { @@ -232,7 +232,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { if (y == pTree->NIL) { pTree->root = z; - } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) { + } else if (pTree->cmprFn(z, y) < 0) { y->left = z; } else { y->right = z; @@ -245,10 +245,10 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { tRBTreePutFix(pTree, z); // update min/max node - if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) { + if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min, z) > 0) { pTree->min = z; } - if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) { + if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max, z) < 0) { pTree->max = z; } pTree->n++; @@ -309,11 +309,11 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { return pNode; } -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { - int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode)); + int32_t c = pTree->cmprFn(pKeyNode, pNode); if (c < 0) { pNode = pNode->left; diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 4ce668e6ba..d93e9fc569 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -145,7 +145,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it tSkipListWLock(pSkipList); void *pData = iterate(iter); - if (pData == NULL) return; + if (pData == NULL) { + tSkipListUnlock(pSkipList); + return; + } // backward to put the first data hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 0a835ec564..c68375894c 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -32,6 +32,10 @@ class TDTestCase: if "community" in selfPath: projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] else: projPath = selfPath[: selfPath.find("tests")] @@ -55,11 +59,10 @@ class TDTestCase: tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") - tdSql.query("select count(*) from (select distinct(tbname) from db.stb)") - tdSql.checkData(0, 0, 10) + tdSql.query("show db.tables") + tdSql.checkRows(10) tdSql.query("select count(*) from db.stb") - if len(tdSql.queryResult): - tdLog.exit("query result is %d" % len(tdSql.queryResult)) + tdSql.checkData(0, 0, 100) def stop(self): tdSql.close() diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json index d4b2aae2fb..da22ef75e2 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json @@ -20,8 +20,10 @@ "super_tables": [{ "name": "stb", "childtable_prefix": "stb_", + "childtable_count": 10, + "insert_rows": 10, "columns": [{"type": "INT"}], "tags": [{"type": "INT"}] }] }] -} \ No newline at end of file +} diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh new file mode 100755 index 0000000000..457c9eae20 --- /dev/null +++ b/tests/script/coverage_test.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +branch= +if [ x$1 != x ];then + branch=$1 + echo "Testing branch: $branch" +else + echo "Please enter branch name as a parameter" + exit 1 +fi + +today=`date +"%Y%m%d"` +TDENGINE_DIR=/root/pxiao/TDengine +TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDengine { + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + + # reset counter + lcov -d . --zerocounters + + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + fi + + git reset --hard + git checkout -- . + git checkout $branch + git clean -dfx + git pull + git submodule update --init --recursive -f + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + echo "rebuild.." + LOCAL_COMMIT=`git rev-parse --short @` + + rm -rf * + if [ "$branch" == "3.0" ]; then + echo "3.0 =============" + cmake -DCOVER=true -DBUILD_TEST=true .. + else + cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null + fi + make -j4 + make install +} + +function runGeneralCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + case=`echo $line | grep sim$ | awk '{print $NF}'` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTestNGCaseOneByOne { + while read -r line; do + if [[ $line =~ ^taostest* ]]; then + case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2` + yaml=`echo $line | cut -d' ' -f 2` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTest { + echo "run Test" + + if [ "$branch" == "3.0" ]; then + echo "start run unit test case ................" + echo " $TDENGINE_DIR/debug " + cd $TDENGINE_DIR/debug + ctest -j12 + echo "3.0 unit test done" + fi + + cd $TDENGINE_DIR/tests/script + + [ -d ../../sim ] && rm -rf ../../sim + [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT + + runGeneralCaseOneByOne jenkins/basic.txt + + sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT + + totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + sed "3G" $TDENGINE_COVERAGE_REPORT + + stopTaosd + echo "run TestNG cases" + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + cd $TDENGINE_DIR/../TestNG/cases + runTestNGCaseOneByOne ../scripts/cases.txt + echo "TestNG cases done" + + cd $TDENGINE_DIR/tests + rm -rf ../sim + /root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT + + + sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT + totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ "$totalPySuccess" -gt "0" ]; then + sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ -z $totalPyFailed ]; then + sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT + else + sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + + echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT + # Test Connector + stopTaosd + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + + cd $TDENGINE_DIR/src/connector/jdbc + mvn clean package > /dev/null 2>&1 + mvn test > jdbc-out.log 2>&1 + tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test C Demo + stopTaosd + $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null & + sleep 10 + yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test waltest + dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'` + walDir=`find $dataDir -name "wal"|head -n1` + echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT + echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT + if [ -n "$walDir" ]; then + yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + fi + + # run Unit Test + echo "Run Unit Test: utilTest, queryTest and cliTest" + #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!" + #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!" + #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!" + + stopTaosd + + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + # collect data + lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + if [ "$branch" == "3.0" ]; then + lcov --remove coverage.info \ + '*/contrib/*' '*/tests/*' '*/test/*'\ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\ + '*/tthread.c' '*/tversion.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + else + lcov --remove coverage.info \ + '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + fi + + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + + # push result to coveralls.io + echo "push result to coveralls.io" + /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + + #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT + #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT + cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function sendReport { + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasod end" +} + +function runTestRandomFail { + exec_random_fail_sh=$1 + default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh + [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1 + + dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py + default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py + [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1 + + runTest NoRandomFail +} + +WORK_DIR=/root/pxiao + +date >> $WORK_DIR/cron.log +echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log + +stopTaosd +buildTDengine + +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh + +runTest + +lcovFunc +#sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 89e26e785b..d8f4a36261 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -155,7 +155,7 @@ ./test.sh -f tsim/parser/select_with_tags.sim ./test.sh -f tsim/parser/selectResNum.sim ./test.sh -f tsim/parser/set_tag_vals.sim -./test.sh -f tsim/parser/single_row_in_tb.sim +# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim ./test.sh -f tsim/parser/sliding.sim ./test.sh -f tsim/parser/slimit_alter_tags.sim ./test.sh -f tsim/parser/slimit.sim @@ -194,7 +194,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -# TD-17919 ./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim # ---- show ---- diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim index 59156080c8..31fc276298 100644 --- a/tests/script/tsim/mnode/basic1.sim +++ b/tests/script/tsim/mnode/basic1.sim @@ -42,6 +42,7 @@ sql_error drop mnode on dnode 1 print =============== create mnode 2 sql create mnode on dnode 2 +print =============== create mnode 2 finished $x = 0 step2: $x = $x + 1 @@ -69,9 +70,10 @@ if $data(2)[2] != follower then goto step2 endi -sleep 2000 print ============ drop mnode 2 sql drop mnode on dnode 2 + +print ============ drop mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 1 then return -1 @@ -109,6 +111,8 @@ sleep 2000 print =============== create mnodes sql create mnode on dnode 2 + +print =============== create mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 2 then return -1 diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index a2b8aa2f96..6739a87b42 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -45,49 +45,12 @@ endi if $data(2)[4] != ready then goto step2 endi - -system sh/exec.sh -n dnode3 -s stop -sql_error create mnode on dnode 3 - -print =============== step3: select * from information_schema.ins_mnodes - -$x = 0 -step3: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi -sql select * from information_schema.ins_mnodes -x step3 -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] -print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] - -$leaderNum = 0 -if $data(1)[2] == leader then - $leaderNum = 1 -endi -if $data(2)[2] == leader then - $leaderNum = 1 -endi -if $leaderNum == 0 then - goto step3 -endi -if $data(3)[2] != offline then - goto step3 -endi -if $data(1)[3] != ready then - goto step3 -endi -if $data(2)[3] != ready then - goto step3 -endi -if $data(3)[3] != creating then - goto step3 +if $data(3)[4] != ready then + goto step2 endi -print =============== step4: start dnode3 -system sh/exec.sh -n dnode3 -s start +print =============== step4: create mnode 3 +sql create mnode on dnode 3 $x = 0 step4: @@ -159,12 +122,11 @@ endi if $data(2)[3] != ready then goto step5 endi -if $data(3)[3] != dropping then - goto step5 -endi print =============== step6: start dnode3 system sh/exec.sh -n dnode3 -s start +sql drop mnode on dnode 1 -x step60 +step60: $x = 0 step6: diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 387f38a717..16e8fa3dfa 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -232,12 +232,11 @@ endi if $leaderNum != 1 then goto step81 endi -if $data(1)[3] != dropping then - goto step81 -endi print =============== step9: start mnode1 and wait it dropped system sh/exec.sh -n dnode1 -s start +sql drop mnode on dnode 1 -x step90 +step90: $x = 0 step91: diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 6fa7e39335..252ef10b1a 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -62,7 +62,16 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data -sql select * from c1 + +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select * from c1 -x steps + sql select * from stb sql select * from stb_1 sql select ts, c1, c2, c3 from c1 diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 11a387ed4d..89b6cadef8 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -158,9 +158,15 @@ print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v -sleep 1000 +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select avg(tbcol) as c from stb -x steps -sql select avg(tbcol) as c from stb sql select avg(tbcol) as c from stb where ts <= 1601481840000 sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 sql select avg(tbcol) as c from stb interval(1m) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1c6aef6286..cc85b8ddc8 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -# unstable python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3 @@ -248,8 +248,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 @@ -302,7 +302,7 @@ python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqCheckData1.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py +#python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqAlterSchema.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 8aafd747af..46eaf6cff1 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -325,24 +325,23 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { - printf(" **************************** How To Use TAB Key ********************************\n"); - printf(" * TDengine Command Line supports pressing TAB key to complete word, *\n"); - printf(" * including database name, table name, function name and keywords. *\n"); - printf(" * Press TAB key anywhere, you'll get surprise. *\n"); - printf(" * KEYBOARD SHORTCUT: *\n"); - printf(" * [ TAB ] ...... Complete the word or show help if no input *\n"); - printf(" * [ Ctrl + A ] ...... move cursor to [A]head of line *\n"); - printf(" * [ Ctrl + E ] ...... move cursor to [E]nd of line *\n"); - printf(" * [ Ctrl + W ] ...... move cursor to line of middle *\n"); - printf(" * [ Ctrl + L ] ...... clean screen *\n"); - printf(" * [ Ctrl + K ] ...... clean after cursor *\n"); - printf(" * [ Ctrl + U ] ...... clean before cursor *\n"); - printf(" * *\n"); + printf(" ****************************** Tab Completion **********************************\n"); + printf(" * The TDengine CLI supports tab completion for a variety of items, *\n"); + printf(" * including database names, table names, function names and keywords. *\n"); + printf(" * The full list of shortcut keys is as follows: *\n"); + printf(" * [ TAB ] ...... complete the current word *\n"); + printf(" * ...... if used on a blank line, display all valid commands *\n"); + printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n"); + printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n"); + printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n"); + printf(" * [ Ctrl + L ] ...... clear the entire screen *\n"); + printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n"); + printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n"); printf(" **********************************************************************************\n\n"); } void showHelp() { - printf("\nThe following are supported commands for TDengine Command Line:"); + printf("\nThe TDengine CLI supports the following commands:"); printf( "\n\ ----- A ----- \n\ diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index fd0ea60323..37d00c0a7c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) { printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); // call auto tab - callbackAutoTab(command, pSql, false); + callbackAutoTab(command, NULL, false); } printf("\r\n"); @@ -998,7 +998,9 @@ void *shellCancelHandler(void *arg) { shell.stop_query = true; } else { #endif - taos_kill_query(shell.conn); + if (shell.conn) { + taos_kill_query(shell.conn); + } #ifdef WEBSOCKET } #endif