diff --git a/README-CN.md b/README-CN.md index 627b35698c..437d671bb9 100644 --- a/README-CN.md +++ b/README-CN.md @@ -107,7 +107,7 @@ sudo yum config-manager --set-enabled Powertools ### macOS ``` -sudo brew install argp-standalone pkgconfig +brew install argp-standalone pkgconfig ``` ### 设置 golang 开发环境 @@ -276,7 +276,7 @@ sudo make install 安装成功后，可以在应用程序中双击 TDengine 图标启动服务，或者在终端中启动 TDengine 服务： ```bash -launchctl start taosd +launchctl start com.tdengine.taosd ``` 用户可以使用 TDengine CLI 来连接 TDengine 服务，在终端中，输入： diff --git a/README.md b/README.md index 00d36efb5a..8d2567a816 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ sudo yum config-manager --set-enabled powertools ### macOS ``` -sudo brew install argp-standalone pkgconfig +brew install argp-standalone pkgconfig ``` ### Setup golang environment @@ -280,7 +280,7 @@ Installing from source code will also configure service management for TDengine. To start the service after installation, double-click the /applications/TDengine to start the program, or in a terminal, use: ```bash -launchctl start taosd +launchctl start com.tdengine.taosd ``` Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: diff --git a/cmake/cmake.platform b/cmake/cmake.platform index e4d440d76e..3e239d2e0c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -45,10 +45,19 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.") - IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") - MESSAGE("Current system arch is 64") + IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") + MESSAGE("Current system arch is arm64") SET(TD_DARWIN_64 TRUE) + SET(TD_DARWIN_ARM64 TRUE) ADD_DEFINITIONS("-D_TD_DARWIN_64") + ADD_DEFINITIONS("-D_TD_DARWIN_ARM64") + ENDIF () + IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") + MESSAGE("Current system arch is x86_64") + SET(TD_DARWIN_64 TRUE) + SET(TD_DARWIN_X64 TRUE) + ADD_DEFINITIONS("-D_TD_DARWIN_64") + ADD_DEFINITIONS("-D_TD_DARWIN_X64") ENDIF () ADD_DEFINITIONS("-DHAVE_UNISTD_H") diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 7bde332c8c..7e441c162a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d58230c + GIT_TAG 217a267 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a1a6cd4519..969724cc8d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -270,7 +270,7 @@ if(${JEMALLOC_ENABLED}) PREFIX "jemalloc" SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls --with-malloc-conf='background_thread:true,metadata_thp:auto' BUILD_COMMAND ${MAKE} ) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index f734e354fb..7800fb1c88 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -189,13 +189,13 @@ After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengin -After the installation is complete, double-click the /applications/TDengine to start the program, or run `launchctl start taosd` to start TDengine Server. +After the installation is complete, double-click the /applications/TDengine to start the program, or run `launchctl start com.tdengine.taosd` to start TDengine Server. The following `launchctl` commands can help you manage TDengine service: -- Start TDengine Server: `launchctl start taosd` +- Start TDengine Server: `launchctl start com.tdengine.taosd` -- Stop TDengine Server: `launchctl stop taosd` +- Stop TDengine Server: `launchctl stop com.tdengine.taosd` - Check TDengine Server status: `launchctl list | grep taosd` diff --git a/docs/en/12-taos-sql/28-recovery.md b/docs/en/12-taos-sql/28-recovery.md index 14ac14f867..e869ffc45f 100644 --- a/docs/en/12-taos-sql/28-recovery.md +++ b/docs/en/12-taos-sql/28-recovery.md @@ -16,10 +16,10 @@ You can use the SHOW CONNECTIONS statement to find the conn_id. ## Terminate a Query ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -You can use the SHOW QUERIES statement to find the query_id. +You can use the SHOW QUERIES statement to find the kill_id. ## Terminate a Transaction diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index d91675a0ab..8053c4d25d 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -675,7 +675,7 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Meaning | Whether to generate core file when server crashes | | Value Range | 0: false, 1: true | | Default Value | 1 | -| Note | The core file is generated under root directory `systemctl/launchctl start taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | +| Note | The core file is generated under root directory `systemctl start taosd`/`launchctl start com.tdengine.taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | ### udf diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 50ee721386..167a68d3c8 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -188,13 +188,13 @@ Active: inactive (dead) -安装后，在应用程序目录下，双击 TDengine 图标来启动程序，也可以运行 `launchctl start taosd` 来启动 TDengine 服务进程。 +安装后，在应用程序目录下，双击 TDengine 图标来启动程序，也可以运行 `launchctl start com.tdengine.taosd` 来启动 TDengine 服务进程。 如下 `launchctl` 命令可以帮助你管理 TDengine 服务： -- 启动服务进程：`launchctl start taosd` +- 启动服务进程：`launchctl start com.tdengine.taosd` -- 停止服务进程：`launchctl stop taosd` +- 停止服务进程：`launchctl stop com.tdengine.taosd` - 查看服务状态：`launchctl list | grep taosd` diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 4c33b5232a..9346c3f763 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -340,7 +340,7 @@ LTRIM(expr) #### RTRIM ```sql -LTRIM(expr) +RTRIM(expr) ``` **功能说明**：返回清除右边空格后的字符串。 diff --git a/docs/zh/12-taos-sql/28-recovery.md b/docs/zh/12-taos-sql/28-recovery.md index 582c373907..b5088e7982 100644 --- a/docs/zh/12-taos-sql/28-recovery.md +++ b/docs/zh/12-taos-sql/28-recovery.md @@ -17,10 +17,10 @@ conn_id 可以通过 `SHOW CONNECTIONS` 获取。 ## 终止查询 ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -query_id 可以通过 `SHOW QUERIES` 获取。 +kill_id 可以通过 `SHOW QUERIES` 获取。 ## 终止事务 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 7da8e9f331..1ab7f4f016 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 + + +## SCHEMALESS 变更 + +| # | **元素** | **
差异性
** | **说明** | +| - | :------- | :-------- | :------- | +| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。 diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 07d5b0692b..b89f625376 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -115,7 +115,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 ## 数据模式映射规则 -本节将说明行协议的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 +本节将说明 InfluxDB 行协议(Line Protocol)的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 超级表名称。tag_set 中的 标签名称为 数据模式中的标签名，field_set 中的名称为列名称。以如下数据为例，说明映射规则： ```json diff --git a/examples/c/demoapi.c b/examples/c/demoapi.c index c3a6d26776..18f1b5a059 100644 --- a/examples/c/demoapi.c +++ b/examples/c/demoapi.c @@ -1,5 +1,17 @@ -// C api call sequence demo -// to compile: gcc -o apidemo apidemo.c -ltaos +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include #include @@ -13,23 +25,23 @@ #include "taos.h" #define debugPrint(fmt, ...) \ - do { if (g_args.debug_print || g_args.verbose_print) \ - fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0) + do { if (g_args.debug_print || g_args.verbose_print) {\ + fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0) #define warnPrint(fmt, ...) \ do { fprintf(stderr, "\033[33m"); \ fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define errorPrint(fmt, ...) \ do { fprintf(stderr, "\033[31m"); \ fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define okPrint(fmt, ...) \ do { fprintf(stderr, "\033[32m"); \ fprintf(stderr, "OK: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) int64_t g_num_of_tb = 2; int64_t g_num_of_rec = 3; @@ -74,10 +86,16 @@ static void prepare_data(TAOS* taos) { res = taos_query(taos, "create database test;"); taos_free_result(res); taosMsleep(100); - taos_select_db(taos, "test"); + if (taos_select_db(taos, "test")) { + errorPrint("%s() LN%d: taos_select_db() failed\n", + __func__, __LINE__); + return; + } char command[1024] = {0}; - sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));"); + sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, " + "bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, " + "city binary(20), dist nchar(20), street binary(20));"); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { okPrint("%s created\n", "meters"); @@ -117,12 +135,11 @@ static void prepare_data(TAOS* taos) { sprintf(command, "insert into t%"PRId64" " "values(%" PRId64 ", %f, %"PRId64", " "'%c%d', '%s%c%d', '%c%d')", - i, 1650000000000+j, (float)j, j, + i, 1650000000000+j, j * 1.0, j, 'a'+(int)j%25, rand(), // "涛思", 'z' - (int)j%25, rand(), "TAOS", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -150,8 +167,7 @@ static void prepare_data(TAOS* taos) { i, 1650000000000+j+1, (float)j, j, 'a'+(int)j%25, rand(), "数据", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -179,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { } if (block) { - warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n"); + warnPrint("%s", "call taos_fetch_block(), " + "don't call taos_fetch_lengths()\n"); int rows = 0; while ((rows = taos_fetch_block(res, &row))) { int *lengths = taos_fetch_lengths(res); @@ -190,7 +207,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d type is %d, no need get offset\n", f, fields[f].type); for (int64_t c = 0; c < rows; c++) { - switch(fields[f].type) { + switch (fields[f].type) { case TSDB_DATA_TYPE_TIMESTAMP: if (taos_is_null(res, c, f)) { printf("col%d, row: %"PRId64" " @@ -199,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %"PRId64"\n", f, c, - *(int64_t*)((char*)(row[f])+c*sizeof(int64_t))); + *(int64_t*)((char*)(row[f]) + +c*sizeof(int64_t))); } break; @@ -211,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %d\n", f, c, - *(int32_t*)((char*)(row[f])+c*sizeof(int32_t))); + *(int32_t*)((char*)(row[f]) + +c*sizeof(int32_t))); } break; @@ -223,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %f\n", f, c, - *(float*)((char*)(row[f])+c*sizeof(float))); + *(float*)((char*)(row[f]) + +c*sizeof(float))); } break; @@ -238,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { if (offsets) { for (int c = 0; c < rows; c++) { if (offsets[c] != -1) { - int length = *(int16_t*)((char*)(row[f]) + offsets[c]); + int length = *(int16_t*)((char*)(row[f]) + + offsets[c]); char *buf = calloc(1, length + 1); - strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length); - printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n", + strncpy(buf, (char *)((char*)(row[f]) + + offsets[c] + 2), length); + printf("row: %d, col: %d, offset: %d, " + "length: %d, content: %s\n", c, f, offsets[c], length, buf); free(buf); } else { - printf("row: %d, col: %d, offset: -1, means content is NULL\n", + printf("row: %d, col: %d, offset: -1, " + "means content is NULL\n", c, f); } } @@ -267,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { int* lengths = taos_fetch_lengths(res); if (lengths) { for (int c = 0; c < num_fields; c++) { - printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n", + printf("row: %"PRId64", col: %d, is_null: %s, " + "length of column %d is %d\n", num_rows, c, taos_is_null(res, num_rows, c)?"True":"False", c, lengths[c]); @@ -277,7 +302,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { __func__, __LINE__, tbname); } - num_rows ++; + num_rows++; } } @@ -322,7 +347,8 @@ int main(int argc, char *argv[]) { #endif TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { - printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); + printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", + taos_errstr(taos)); exit(1); } diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 0e22985059..fd0521af3b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -93,6 +93,7 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in // query client extern int32_t tsQueryPolicy; +extern int32_t tsQueryRspPolicy; extern int32_t tsQuerySmaOptimize; extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 0d43539629..cdb1642a5c 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -30,9 +30,10 @@ typedef struct SMnode SMnode; typedef struct { int32_t dnodeId; - bool standby; bool deploy; - SReplica replica; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; SMsgCb msgCb; } SMnodeOpt; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c179d69960..7d95e840e9 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -54,6 +54,9 @@ typedef enum { #define QUERY_POLICY_QNODE 3 #define QUERY_POLICY_CLIENT 4 +#define QUERY_RSP_POLICY_DELAY 0 +#define QUERY_RSP_POLICY_QUICK 1 + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision @@ -230,7 +233,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, - void* (*mallocFp)(int32_t)); + void* (*mallocFp)(int64_t)); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 6322c6a1e9..ef8e80b57f 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -43,7 +43,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); -int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow); +int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows); /* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 411a63a379..0f8220f19c 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -211,7 +211,7 @@ typedef struct SSyncInfo { int32_t syncInit(); void syncCleanUp(); -int64_t syncOpen(const SSyncInfo* pSyncInfo); +int64_t syncOpen(SSyncInfo* pSyncInfo); void syncStart(int64_t rid); void syncStop(int64_t rid); int32_t syncSetStandby(int64_t rid); @@ -233,7 +233,7 @@ const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot); -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg); // build SRpcMsg, need to call syncPropose with SRpcMsg int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 56c5750475..53edcd4fee 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -123,9 +123,9 @@ void rpcCleanup(); void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void rpcCloseImpl(void *); -void *rpcMallocCont(int32_t contLen); +void *rpcMallocCont(int64_t contLen); void rpcFreeCont(void *pCont); -void *rpcReallocCont(void *ptr, int32_t contLen); +void *rpcReallocCont(void *ptr, int64_t contLen); // Because taosd supports multi-process mode // These functions should not be used on the server side diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 3146590d6c..7ec8366ee4 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -29,12 +29,12 @@ extern "C" { #define free FREE_FUNC_TAOS_FORBID #endif -void *taosMemoryMalloc(int32_t size); -void *taosMemoryCalloc(int32_t num, int32_t size); -void *taosMemoryRealloc(void *ptr, int32_t size); +void *taosMemoryMalloc(int64_t size); +void *taosMemoryCalloc(int64_t num, int64_t size); +void *taosMemoryRealloc(void *ptr, int64_t size); void *taosMemoryStrDup(const char *ptr); void taosMemoryFree(void *ptr); -int32_t taosMemorySize(void *ptr); +int64_t taosMemorySize(void *ptr); void taosPrintBackTrace(); #define taosMemoryFreeClear(ptr) \ diff --git a/include/util/trbtree.h b/include/util/trbtree.h index f6d37e3d75..7c93c8dd8d 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -26,7 +26,7 @@ typedef struct SRBTree SRBTree; typedef struct SRBTreeNode SRBTreeNode; typedef struct SRBTreeIter SRBTreeIter; -typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); +typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *); // SRBTree ============================================= #define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min) @@ -36,7 +36,7 @@ void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn); SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z); void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode); // SRBTreeIter ============================================= #define tRBTreeIterCreate(tree, ascend) \ @@ -53,8 +53,6 @@ struct SRBTreeNode { SRBTreeNode *right; }; -#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) - struct SRBTree { tRBTreeCmprFn cmprFn; int64_t n; diff --git a/packaging/tools/TDengine b/packaging/tools/TDengine index 3025a5639f..c0b80564c4 100755 --- a/packaging/tools/TDengine +++ b/packaging/tools/TDengine @@ -17,12 +17,12 @@ EOF taosd_status=`Launchctl list | grep taosd | head -n 1 | awk '{print $1}'` if [ "$taosd_status"x = "-"x ]; then - launchctl start taosd + launchctl start com.tdengine.taosd showAlertMessage "Taosd is running!" "TDengine" "ok" "note" else choose_result=`showAlertMessage "Taosd is running!\nDo you want to close it?" "TDengine" "yes,cancel" "stop"` if [ "$choose_result"x = "button returned:yes"x ]; then - launchctl stop taosd + launchctl stop com.tdengine.taosd fi fi diff --git a/packaging/tools/TDengine.pkgproj b/packaging/tools/TDengine.pkgproj new file mode 100644 index 0000000000..7aaac62735 --- /dev/null +++ b/packaging/tools/TDengine.pkgproj @@ -0,0 +1,882 @@ + + + + + PACKAGES + + + MUST-CLOSE-APPLICATION-ITEMS + + MUST-CLOSE-APPLICATIONS + + PACKAGE_FILES + + DEFAULT_INSTALL_LOCATION + / + HIERARCHY + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Applications + PATH_TYPE + 0 + PERMISSIONS + 509 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Application Support + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Automator + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Documentation + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Extensions + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Filesystems + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Frameworks + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Input Methods + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Internet Plug-Ins + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Keyboard Layouts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchAgents + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchDaemons + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PreferencePanes + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Preferences + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 80 + PATH + Printers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PrivilegedHelperTools + PATH_TYPE + 0 + PERMISSIONS + 1005 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickLook + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickTime + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Screen Savers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Scripts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Services + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Widgets + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + Library + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 0 + PATH + Shared + PATH_TYPE + 0 + PERMISSIONS + 1023 + TYPE + 1 + UID + 0 + + + GID + 80 + PATH + Users + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + / + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + PAYLOAD_TYPE + 0 + PRESERVE_EXTENDED_ATTRIBUTES + + SHOW_INVISIBLE + + SPLIT_FORKS + + TREAT_MISSING_FILES_AS_WARNING + + VERSION + 5 + + PACKAGE_SCRIPTS + + POSTINSTALL_PATH + + PATH + /opt/taos/TDengine/packaging/tools/post.sh + PATH_TYPE + 0 + + PREINSTALL_PATH + + PATH_TYPE + 0 + + RESOURCES + + + CHILDREN + + GID + 0 + PATH + /opt/tdengine + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 3 + UID + 0 + + + + PACKAGE_SETTINGS + + AUTHENTICATION + 1 + CONCLUSION_ACTION + 0 + FOLLOW_SYMBOLIC_LINKS + + IDENTIFIER + com.taosdata.pkg.pkgtaos + LOCATION + 0 + NAME + TDengine + OVERWRITE_PERMISSIONS + + PAYLOAD_SIZE + -1 + REFERENCE_PATH + + RELOCATABLE + + USE_HFS+_COMPRESSION + + VERSION + 3.0.1.4 + + TYPE + 0 + UUID + 48F6B249-AF56-46E3-B75A-AEED5858A764 + + + PROJECT + + PROJECT_COMMENTS + + NOTES + + + + PROJECT_PRESENTATION + + BACKGROUND + + APPAREANCES + + DARK_AQUA + + LIGHT_AQUA + + + SHARED_SETTINGS_FOR_ALL_APPAREANCES + + + INSTALLATION TYPE + + HIERARCHIES + + INSTALLER + + LIST + + + CHILDREN + + DESCRIPTION + + OPTIONS + + HIDDEN + + STATE + 1 + + PACKAGE_UUID + 48F6B249-AF56-46E3-B75A-AEED5858A764 + TITLE + + TYPE + 0 + UUID + 7ED88D2C-D55C-46FF-99CB-73313ACAD73D + + + REMOVED + + + + MODE + 0 + + INSTALLATION_STEPS + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewIntroductionController + INSTALLER_PLUGIN + Introduction + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewReadMeController + INSTALLER_PLUGIN + ReadMe + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewLicenseController + INSTALLER_PLUGIN + License + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewDestinationSelectController + INSTALLER_PLUGIN + TargetSelect + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewInstallationTypeController + INSTALLER_PLUGIN + PackageSelection + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewInstallationController + INSTALLER_PLUGIN + Install + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewSummaryController + INSTALLER_PLUGIN + Summary + LIST_TITLE_KEY + InstallerSectionTitle + + + INTRODUCTION + + LOCALIZATIONS + + + LANGUAGE + English + VALUE + + PATH + /opt/taos/TDengine/packaging/tools/mac_before_install.txt + PATH_TYPE + 0 + + + + + LICENSE + + LOCALIZATIONS + + MODE + 0 + + README + + LOCALIZATIONS + + + SUMMARY + + LOCALIZATIONS + + + TITLE + + LOCALIZATIONS + + + LANGUAGE + English + VALUE + TDengine + + + + + PROJECT_REQUIREMENTS + + LIST + + RESOURCES + + ROOT_VOLUME_ONLY + + + PROJECT_SETTINGS + + BUILD_FORMAT + 0 + BUILD_PATH + + PATH + /opt/taos/TDengine/release + PATH_TYPE + 0 + + EXCLUDED_FILES + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .DS_Store + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .DS_Store files + PROXY_TOOLTIP + Remove ".DS_Store" files created by the Finder. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .pbdevelopment + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .pbdevelopment files + PROXY_TOOLTIP + Remove ".pbdevelopment" files created by ProjectBuilder or Xcode. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + CVS + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .cvsignore + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .cvspass + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .svn + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .git + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .gitignore + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove SCM metadata + PROXY_TOOLTIP + Remove helper files and folders used by the CVS, SVN or Git Source Code Management systems. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + classes.nib + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + designable.db + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + info.nib + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Optimize nib files + PROXY_TOOLTIP + Remove "classes.nib", "info.nib" and "designable.nib" files within .nib bundles. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + Resources Disabled + TYPE + 1 + + + PROTECTED + + PROXY_NAME + Remove Resources Disabled folders + PROXY_TOOLTIP + Remove "Resources Disabled" folders. + STATE + + + + SEPARATOR + + + + NAME + TDengine-client-3.0.1.4-macOS-arm64 + PAYLOAD_ONLY + + TREAT_MISSING_PRESENTATION_DOCUMENTS_AS_WARNING + + + + TYPE + 0 + VERSION + 2 + + diff --git a/packaging/tools/com.taosdata.taosd.plist b/packaging/tools/com.taosdata.taosd.plist index ca9b8c5ef1..0bf3135ac3 100644 --- a/packaging/tools/com.taosdata.taosd.plist +++ b/packaging/tools/com.taosdata.taosd.plist @@ -3,7 +3,7 @@ Label - taosd + com.tdengine.taosd ProgramArguments /usr/local/bin/taosd diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index f70ea7161f..b33cd478b8 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,5 +1,5 @@ TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. To configure TDengine : edit /etc/taos/taos.cfg -To start service : launchctl start taosd +To start service : launchctl start com.tdengine.taosd To access TDengine : use taos in shell \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index b8796ddf09..b56a5a203d 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -615,7 +615,7 @@ function update_TDengine() { if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" @@ -666,7 +666,7 @@ function install_TDengine() { if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 29d4ac017c..b729513b97 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -18,6 +18,7 @@ if [ "$osType" != "Darwin" ]; then script_dir=$(dirname $(readlink -f "$0")) verNumber="" lib_file_ext="so" + lib_file_ext_1="so.1" bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" @@ -29,6 +30,7 @@ else script_dir=${source_dir}/packaging/tools verNumber=`ls tdengine/driver | grep -E "libtaos\.[0-9]\.[0-9]" | sed "s/libtaos.//g" | sed "s/.dylib//g" | head -n 1` lib_file_ext="dylib" + lib_file_ext_1="1.dylib" bin_link_dir="/usr/local/bin" lib_link_dir="/usr/local/lib" @@ -134,14 +136,14 @@ function install_lib() { [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} + ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib_link_dir}/libtaosws.${lib_file_ext} ||: - if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} || : [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} || : fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6172fc9bc8..bace0bc869 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -117,7 +117,7 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : + [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index e8e3c87849..0c445c7fbf 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -19,6 +19,13 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) +if(TD_DARWIN_ARM64) + target_link_libraries( + taos + PRIVATE "-arch x86_64" + ) +endif() + if(TD_WINDOWS) INCLUDE_DIRECTORIES(jni/windows) INCLUDE_DIRECTORIES(jni/windows/win32) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7b437e4769..940dd745c8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1036,30 +1036,38 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->type = pQuery->msgType; SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); + SPlanContext cxt = {.queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite, - .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, - .pUser = pRequest->pTscObj->user, - .sysInfo = pRequest->pTscObj->sysInfo, - .allocatorId = pRequest->allocatorRefId}; - SQueryPlan* pDag = NULL; - int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); - if (code) { - tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); - } else { - pRequest->body.subplanNum = pDag->numOfSubplans; + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user, + .sysInfo = pRequest->pTscObj->sysInfo, + .allocatorId = pRequest->allocatorRefId}; + + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SQueryPlan* pDag = NULL; + + int64_t st = taosGetTimestampUs(); + int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); + if (code) { + tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + pRequest->body.subplanNum = pDag->numOfSubplans; + } + + pRequest->metric.planEnd = taosGetTimestampUs(); + if (code == TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self, + (pRequest->metric.planEnd - st)/1000.0, pRequest->requestId); } - - pRequest->metric.planEnd = taosGetTimestampUs(); - - if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { - SArray* pNodeList = NULL; - buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { + SArray* pNodeList = NULL; + buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter, .requestId = pRequest->requestId, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index fa0a4ac743..6126817ece 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -221,7 +221,9 @@ void taos_kill_query(TAOS *taos) { int64_t rid = *(int64_t *)taos; STscObj *pTscObj = acquireTscObj(rid); - stopAllRequests(pTscObj->pRequests); + if (pTscObj) { + stopAllRequests(pTscObj->pRequests); + } releaseTscObj(rid); } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c5633e73d0..24da5f7b70 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2477,11 +2477,12 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { } else { pParam->request->body.resInfo.numOfRows += info->affectedRows; } + // unlock + taosThreadSpinUnlock(&pParam->lock); + if (pParam->cnt == pParam->total) { tsem_post(&pParam->sem); } - taosThreadSpinUnlock(&pParam->lock); - // unlock uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); info->cost.endTime = taosGetTimestampUs(); info->cost.code = code; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 66f992f05f..922cab23c8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -972,6 +972,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; + tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); + for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1297,7 +1299,8 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { pParam->code = code; if (code != 0) { - tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); + tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d, code %x", tmq->consumerId, + pParam->async, code); goto END; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69174476a..21a52a4b57 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) { int32_t n = 0; char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t* length = taos_fetch_lengths(pRes); - for (int32_t i = 0; i < numOfFields; ++i) { - printf("(%d):%d ", i, length[i]); - } - printf("\n"); - - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - memset(str, 0, sizeof(str)); +// int32_t* length = taos_fetch_lengths(pRes); +// for(int32_t i = 0; i < numOfFields; ++i) { +// printf("(%d):%d " , i, length[i]); +// } +// printf("\n"); +// +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// memset(str, 0, sizeof(str)); } } @@ -102,17 +102,6 @@ void queryCallback(void* param, void* res, int32_t code) { taos_fetch_raw_block_a(res, fetchCallback, param); } -void queryCallback1(void* param, void* res, int32_t code) { - if (code != TSDB_CODE_SUCCESS) { - printf("failed to execute, reason:%s\n", taos_errstr(res)); - } - - taos_free_result(res); - - printf("exec query:\n"); - taos_query_a(param, "select * from tm1", queryCallback, param); -} - void createNewTable(TAOS* pConn, int32_t index) { char str[1024] = {0}; sprintf(str, "create table tu%d using st2 tags(%d)", index, index); @@ -123,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for (int32_t i = 0; i < 3280; i += 20) { + for(int32_t i = 0; i < 10000; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -141,10 +130,49 @@ void createNewTable(TAOS* pConn, int32_t index) { taos_free_result(p); } } + +void *queryThread(void *arg) { + TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to db, reason:%s", taos_errstr(pConn)); + return NULL; + } + + int64_t el = 0; + + for (int32_t i = 0; i < 5000000; ++i) { + int64_t st = taosGetTimestampUs(); + TAOS_RES* pRes = taos_query(pConn, + "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;"); + if (taos_errno(pRes) != 0) { + printf("failed, reason:%s\n", taos_errstr(pRes)); + } else { + printResult(pRes); + } + + taos_free_result(pRes); + el += (taosGetTimestampUs() - st); + if (i % 1000 == 0 && i != 0) { + printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000)); + } + } + + taos_close(pConn); + return NULL; +} + +static int32_t numOfThreads = 1; } // namespace int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + numOfThreads = atoi(argv[1]); + } + + numOfThreads = TMAX(numOfThreads, 1); + printf("the runing threads is:%d", numOfThreads); + return RUN_ALL_TESTS(); } @@ -664,7 +692,6 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } -#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -676,13 +703,14 @@ TEST(testCase, projection_query_tables) { // } // taos_free_result(pRes); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu"); taos_free_result(pRes); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } + taos_free_result(pRes); pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); @@ -722,6 +750,16 @@ TEST(testCase, projection_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif + +TEST(testCase, tsbs_perf_test) { + TdThread qid[20] = {0}; + + for(int32_t i = 0; i < numOfThreads; ++i) { + taosThreadCreate(&qid[i], NULL, queryThread, NULL); + } + getchar(); +} #if 0 TEST(testCase, projection_query_stables) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 88df54a9ea..56daa4075a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -90,6 +90,7 @@ bool tsSmlDataFormat = false; // query int32_t tsQueryPolicy = 1; +int32_t tsQueryRspPolicy = 0; int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -350,6 +351,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1; if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; @@ -728,6 +730,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; + tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index a4c37dd334..17cbde437b 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -41,8 +41,8 @@ typedef struct SMnodeMgmt { } SMnodeMgmt; // mmFile.c -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); +int32_t mmReadFile(const char *path, SMnodeOpt *pOption); +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption); // mmHandle.c SArray *mmGetMsgHandles(); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 27a35ae17a..c5ddb9f021 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { +int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 4096; @@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { char file[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { code = 0; @@ -50,38 +50,69 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { dError("failed to read %s since deployed not found", file); goto _OVER; } - *pDeployed = deployed->valueint; + pOption->deploy = deployed->valueint; - cJSON *id = cJSON_GetObjectItem(root, "id"); - if (id) { - if (id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); + cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex"); + if (selfIndex) { + if (selfIndex->type != cJSON_Number) { + dError("failed to read %s since selfIndex not found", file); goto _OVER; } - if (pReplica) { - pReplica->id = id->valueint; - } + pOption->selfIndex = selfIndex->valueint; } - cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn"); - if (fqdn) { - if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); + cJSON *replicas = cJSON_GetObjectItem(root, "replicas"); + if (replicas) { + if (replicas->type != cJSON_Array) { + dError("failed to read %s since replicas not found", file); goto _OVER; } - if (pReplica) { - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - } - } - cJSON *port = cJSON_GetObjectItem(root, "port"); - if (port) { - if (port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); + int32_t numOfReplicas = cJSON_GetArraySize(replicas); + if (numOfReplicas <= 0) { + dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas); goto _OVER; } - if (pReplica) { - pReplica->port = (uint16_t)port->valueint; + pOption->numOfReplicas = numOfReplicas; + + for (int32_t i = 0; i < numOfReplicas; ++i) { + SReplica *pReplica = pOption->replicas + i; + + cJSON *replica = cJSON_GetArrayItem(replicas, i); + if (replica == NULL) break; + + cJSON *id = cJSON_GetObjectItem(replica, "id"); + if (id) { + if (id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->id = id->valueint; + } + } + + cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn"); + if (fqdn) { + if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto _OVER; + } + if (pReplica) { + tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + } + } + + cJSON *port = cJSON_GetObjectItem(replica, "port"); + if (port) { + if (port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->port = (uint16_t)port->valueint; + } + } } } @@ -92,18 +123,18 @@ _OVER: if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); if (code == 0) { - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy); } terrno = code; return code; } -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) { +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - if (pReplica != NULL && pReplica->id > 0) { - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port); + if (pOption->deploy && pOption->numOfReplicas > 0) { + len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex); + len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n"); + + for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + const SReplica *pReplica = pOption->replicas + i; + if (pReplica != NULL && pReplica->id > 0) { + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + } + if (i < pOption->numOfReplicas - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }],\n"); + } + } } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy); len += snprintf(content + len, maxLen - len, "}\n"); taosWriteFile(pFile, content, len); @@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) return -1; } - dDebug("successed to write %s, deployed:%d", realfile, deployed); + dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy); return 0; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index ec761e6441..fc45dbf15f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - if (createReq.replica != 1) { + SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1}; + memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (createReq.replicas[i].id == pInput->pData->dnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { terrno = TSDB_CODE_INVALID_OPTION; - dGError("failed to create mnode since %s", terrstr()); + dGError("failed to create mnode since %s, selfIndex is -1", terrstr()); return -1; } - bool deployed = true; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - bool deployed = false; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, NULL, deployed) != 0) { + SMnodeOpt option = {.deploy = false}; + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 49207225a5..e7f71ad420 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) { } static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - if (mmReadFile(&mgmt, NULL, required) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pInput->path, &option) != 0) { return -1; } - if (!(*required)) { + if (!option.deploy) { *required = mmDeployRequired(pInput); + } else { + *required = true; } return 0; } static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { - pOption->standby = false; pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica.id = 1; - pOption->replica.port = tsServerPort; - tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->selfIndex = 0; + pOption->numOfReplicas = 1; + pOption->replicas[0].id = 1; + pOption->replicas[0].port = tsServerPort; + tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } -static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { - pOption->standby = false; +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pReplica->id > 0) { - pOption->standby = true; - pOption->replica = *pReplica; - } } static void mmClose(SMnodeMgmt *pMgmt) { @@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - bool deployed = false; - SReplica replica = {0}; - if (mmReadFile(pMgmt, &replica, &deployed) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pMgmt->path, &option) != 0) { dError("failed to read file since %s", terrstr()); mmClose(pMgmt); return -1; } - SMnodeOpt option = {0}; - if (!deployed) { + if (!option.deploy) { dInfo("mnode start to deploy"); pMgmt->pData->dnodeId = 1; mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); - mmBuildOptionForOpen(pMgmt, &replica, &option); + mmBuildOptionForOpen(pMgmt, &option); } pMgmt->pMnode = mndOpen(pMgmt->path, &option); @@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (!deployed || replica.id > 0) { - deployed = true; - if (mmWriteFile(pMgmt, NULL, deployed) != 0) { + if (option.numOfReplicas > 0) { + option.deploy = true; + option.numOfReplicas = 0; + if (mmWriteFile(pMgmt->path, &option) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index d2b071ec61..849d3ef390 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -72,7 +72,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { mndPostProcessQueryMsg(pMsg); } - dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); + dGTrace("msg:%p is freed, code:%s", pMsg, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 71315a39fd..f55e830a44 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -87,12 +87,13 @@ typedef struct { typedef struct { tsem_t syncSem; int64_t sync; - SReplica replica; int32_t errCode; int32_t transId; SRWLatch lock; - int8_t standby; int8_t leaderTransferFinish; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; } SSyncMgmt; typedef struct { @@ -130,11 +131,10 @@ typedef struct SMnode { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); int64_t mndGenerateUid(const char *name, int32_t len); -int32_t mndAcquireRpcRef(SMnode *pMnode); -void mndReleaseRpcRef(SMnode *pMnode); -void mndSetRestore(SMnode *pMnode, bool restored); -void mndSetStop(SMnode *pMnode); -bool mndGetStop(SMnode *pMnode); +void mndSetRestored(SMnode *pMnode, bool restored); +bool mndGetRestored(SMnode *pMnode); +void mndSetStop(SMnode *pMnode); +bool mndGetStop(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index cb9d70d5ee..993efbcd08 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -24,7 +24,7 @@ extern "C" { int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); -bool mndIsMaster(SMnode *pMnode); +bool mndIsLeader(SMnode *pMnode); int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId); void mndSyncStart(SMnode *pMnode); void mndSyncStop(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6670211d78..3b31873857 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true); if (pShow->pIter == NULL) break; if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9ade39e8e7..fba1fd94d6 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { goto _OVER; } else { pDnode->accessTimes++; - mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); + mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } } @@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } pDnode->lastAccessTime = curMs; + pDnode->accessTimes++; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index c8459fb53b..5fb23b045c 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -43,6 +43,37 @@ #include "mndUser.h" #include "mndVgroup.h" +static inline int32_t mndAcquireRpc(SMnode *pMnode) { + int32_t code = 0; + taosThreadRwlockRdlock(&pMnode->lock); + if (pMnode->stopped) { + terrno = TSDB_CODE_APP_NOT_READY; + code = -1; + } else if (!mndIsLeader(pMnode)) { + code = -1; + } else { +#if 1 + atomic_add_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is acquired, ref:%d", ref); +#endif + } + taosThreadRwlockUnlock(&pMnode->lock); + return code; +} + +static inline void mndReleaseRpc(SMnode *pMnode) { + taosThreadRwlockRdlock(&pMnode->lock); +#if 1 + atomic_sub_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is released, ref:%d", ref); +#endif + taosThreadRwlockUnlock(&pMnode->lock); +} + static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) { static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->msgCb = pOption->msgCb; pMnode->selfDnodeId = pOption->dnodeId; - pMnode->syncMgmt.replica = pOption->replica; - pMnode->syncMgmt.standby = pOption->standby; + pMnode->syncMgmt.selfIndex = pOption->selfIndex; + pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas; + memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas)); } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) { mError("failed to deploy sdb while start mnode"); return -1; } - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } grantReset(pMnode, TSDB_GRANT_ALL, 0); @@ -570,23 +602,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } - if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; + if (mndAcquireRpc(pMsg->info.node) == 0) return 0; if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { return -1; } - SEpSet epSet = {0}; - mndGetMnodeEpSet(pMsg->info.node, &epSet); + SEpSet epSet = {0}; + SMnode *pMnode = pMsg->info.node; + mndGetMnodeEpSet(pMnode, &epSet); const STraceId *trace = &pMsg->info.traceId; - mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); + mDebug( + "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s " + "numOfEps:%d inUse:%d", + pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync), + syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); if (epSet.numOfEps > 0) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { - mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); } int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); @@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); int32_t code = (*fp)(pMsg); - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mGTrace("msg:%p, won't response immediately since in progress", pMsg); @@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) { int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { - if (mndAcquireRpcRef(pMnode) != 0) return -1; + if (mndAcquireRpc(pMnode) != 0) return -1; SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); @@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc)); if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || pStbInfo->stbs == NULL) { - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return -1; } @@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->timeseries_total = INT32_MAX; } - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return 0; } @@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; } -int32_t mndAcquireRpcRef(SMnode *pMnode) { - int32_t code = 0; - taosThreadRwlockRdlock(&pMnode->lock); - if (pMnode->stopped) { - mTrace("mnode not running"); - terrno = TSDB_CODE_APP_NOT_READY; - code = -1; - } else if (!mndIsMaster(pMnode)) { - mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored); - code = -1; - } else { - int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is acquired, ref:%d", ref); - } - taosThreadRwlockUnlock(&pMnode->lock); - return code; -} - -void mndReleaseRpcRef(SMnode *pMnode) { - taosThreadRwlockRdlock(&pMnode->lock); - int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is released, ref:%d", ref); - taosThreadRwlockUnlock(&pMnode->lock); -} - -void mndSetRestore(SMnode *pMnode, bool restored) { +void mndSetRestored(SMnode *pMnode, bool restored) { if (restored) { taosThreadRwlockWrlock(&pMnode->lock); pMnode->restored = true; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d0440359ff..c27241317c 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -36,6 +36,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq); static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); +static void mndReloadSyncConfig(SMnode *pMnode); int32_t mndInitMnode(SMnode *pMnode) { SSdbTable table = { @@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { } pObj->state = TAOS_SYNC_STATE_ERROR; + mndReloadSyncConfig(pSdb->pMnode); return 0; } @@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; + mndReloadSyncConfig(pSdb->pMnode); + return 0; } @@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pIter == NULL) break; if (pObj->id == pMnode->selfDnodeId) { - if (mndIsMaster(pMnode)) { + if (mndIsLeader(pMnode)) { pEpSet->inUse = pEpSet->numOfEps; } else { pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes; @@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pEpSet->numOfEps == 0) { syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); } + + if (pEpSet->inUse >= pEpSet->numOfEps) { + pEpSet->inUse = 0; + } } static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { @@ -274,13 +282,72 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod return 0; } +static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); + + STransAction action = { + .epSet = *pCreateEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_CREATE_MNODE, + .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + +static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); + + STransAction action = { + .epSet = *pAlterEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_MND_ALTER_MNODE, + .acceptableCode = 0, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { + int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq); + + STransAction action = { + .epSet = *pDroprEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_DROP_MNODE, + .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; SDCreateMnodeReq createReq = {0}; - SEpSet alterEpset = {0}; SEpSet createEpset = {0}; while (1) { @@ -288,75 +355,25 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); if (pIter == NULL) break; - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } + createReq.replicas[numOfReplicas].id = pMObj->id; + createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); numOfReplicas++; sdbRelease(pSdb, pMObj); } - alterReq.replica = numOfReplicas + 1; - alterReq.replicas[numOfReplicas].id = pDnode->id; - alterReq.replicas[numOfReplicas].port = pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.numOfEps = numOfReplicas + 1; - alterEpset.eps[numOfReplicas].port = pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - createReq.replica = 1; - createReq.replicas[0].id = pDnode->id; - createReq.replicas[0].port = pDnode->port; - memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createReq.replica = numOfReplicas + 1; + createReq.replicas[numOfReplicas].id = pDnode->id; + createReq.replicas[numOfReplicas].port = pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createEpset.inUse = 0; createEpset.numOfEps = 1; createEpset.eps[0].port = pDnode->port; memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); - - STransAction action = { - .epSet = createEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_CREATE_MNODE, - .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } + if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; return 0; } @@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); + if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -459,107 +476,28 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO } static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; - SDDropMnodeReq dropReq = {0}; - SSetStandbyReq standbyReq = {0}; - SEpSet alterEpset = {0}; - SEpSet dropEpSet = {0}; - - while (1) { - SMnodeObj *pMObj = NULL; - pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); - if (pIter == NULL) break; - if (pMObj->id == pObj->id) { - sdbRelease(pSdb, pMObj); - continue; - } - - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } - - numOfReplicas++; - sdbRelease(pSdb, pMObj); - } - - alterReq.replica = numOfReplicas; - alterEpset.numOfEps = numOfReplicas; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + int32_t numOfReplicas = 0; + SDDropMnodeReq dropReq = {0}; + SEpSet dropEpSet = {0}; dropReq.dnodeId = pDnode->id; dropEpSet.numOfEps = 1; dropEpSet.eps[0].port = pDnode->port; memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - standbyReq.dnodeId = pDnode->id; - standbyReq.standby = 1; - - { - int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq); - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(MNODE_HANDLE); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_SYNC_SET_MNODE_STANDBY, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_DROP_MNODE, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); + if (totalMnodes == 2) { + mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + } else if (totalMnodes == 3) { + mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes); + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + } else { + return -1; } return 0; @@ -567,7 +505,6 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { if (pObj == NULL) return 0; - if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; if (mndTransAppendNullLog(pTrans) != 0) return -1; @@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { - pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true); if (pShow->pIter == NULL) break; cols = 0; @@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { } static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { +#if 1 + return 0; +#else SMnode *pMnode = pReq->info.node; SDAlterMnodeReq alterReq = {0}; @@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { return -1; } + SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1}; + memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { + mInfo("alter mnode not processed since selfIndex is -1", terrstr()); + return 0; + } + + if (mndWriteFile(pMnode->path, &option) != 0) { + mError("failed to write mnode file since %s", terrstr()); + return -1; + } + SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1}; for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodePort = alterReq.replicas[i].port; - if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i; + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + cfg.myIndex = i; + } } if (cfg.myIndex == -1) { mError("failed to alter mnode since myindex is -1"); return -1; } else { - mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex); + mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); } } - mInfo("trans:-1, sync reconfig will be proposed"); - - SSyncMgmt *pMgmt = &pMnode->syncMgmt; - pMgmt->standby = 0; - int32_t code = syncReconfig(pMgmt->sync, &cfg); + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); if (code != 0) { - mError("trans:-1, failed to propose sync reconfig since %s", terrstr()); - return code; + mError("failed to sync reconfig since %s", terrstr()); } else { - pMgmt->errCode = 0; - taosWLockLatch(&pMgmt->lock); - pMgmt->transId = -1; - taosWUnLockLatch(&pMgmt->lock); - tsem_wait(&pMgmt->syncSem); - mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode)); - terrno = pMgmt->errCode; - return pMgmt->errCode; + mInfo("alter mnode sync success"); + } + + return code; +#endif +} + +static void mndReloadSyncConfig(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + SMnodeObj *pObj = NULL; + ESdbStatus objStatus = 0; + void *pIter = NULL; + bool hasUpdatingMnode = false; + SSyncCfg cfg = {.myIndex = -1}; + + while (1) { + pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false); + if (pIter == NULL) break; + if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) { + mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); + hasUpdatingMnode = true; + } + + if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { + SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; + tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pObj->pDnode->port; + if (pObj->pDnode->id == pMnode->selfDnodeId) { + cfg.myIndex = cfg.replicaNum; + } + cfg.replicaNum++; + } + + sdbReleaseLock(pSdb, pObj, false); + } + + if (cfg.myIndex == -1) { + mInfo("vgId:1, mnode not reload since selfIndex is -1"); + return; + } + + if (!mndGetRestored(pMnode)) { + mInfo("vgId:1, mnode not reload since restore not finished"); + return; + } + + if (hasUpdatingMnode) { + mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); + for (int32_t i = 0; i < cfg.replicaNum; ++i) { + SNodeInfo *pNode = &cfg.nodeInfo[i]; + mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + } + + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); + if (code != 0) { + mError("vgId:1, failed to reconfig mnode sync since %s", terrstr()); + } else { + mInfo("vgId:1, reconfig mnode sync success"); + } } } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0e4dd9fc3c..866a3fa8b9 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { if (!pMnode->deploy) { mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); mndTransPullup(pMnode); - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } else { mInfo("vgId:1, sync restore finished"); } @@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) { snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); syncInfo.pWal = pMnode->pWal; syncInfo.pFsm = mndSyncMakeFsm(pMnode); - syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT; - mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby); - if (pMgmt->standby || pMgmt->replica.id > 0) { - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = 1; - pCfg->myIndex = 0; - SNodeInfo *pNode = &pCfg->nodeInfo[0]; - tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); - pNode->nodePort = pMgmt->replica.port; - mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort); + mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas); + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = pMgmt->numOfReplicas; + pCfg->myIndex = pMgmt->selfIndex; + for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pMgmt->replicas[i].port; + mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); @@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) { setPingTimerMS(pMgmt->sync, 5000); setElectTimerMS(pMgmt->sync, 3000); setHeartbeatTimerMS(pMgmt->sync, 500); - /* - setElectTimerMS(pMgmt->sync, 600); - setHeartbeatTimerMS(pMgmt->sync, 300); - */ mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; @@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); syncStart(pMgmt->sync); - mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); + mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync); } void mndSyncStop(SMnode *pMnode) { @@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) { taosWUnLockLatch(&pMnode->syncMgmt.lock); } -bool mndIsMaster(SMnode *pMnode) { +bool mndIsLeader(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; if (!syncIsReady(pMgmt->sync)) { @@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) { return false; } - if (!pMnode->restored) { + if (!mndGetRestored(pMnode)) { terrno = TSDB_CODE_APP_NOT_READY; return false; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d30cc8f9ca..a64b07fdbd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -53,7 +53,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); -static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } +static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index f922baf329..be93bd2f59 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey); * @param pObj The object of the row. */ void sdbRelease(SSdb *pSdb, void *pObj); +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock); /** * @brief Traverse a sdb table @@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status); +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock); /** * @brief Cancel a traversal diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index ecdf8c71a7..0fee0fbca1 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { taosThreadRwlockUnlock(pLock); } -void sdbRelease(SSdb *pSdb, void *pObj) { +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { if (pObj == NULL) return; SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); if (pRow->type >= SDB_MAX) return; TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + if (lock) { + taosThreadRwlockWrlock(pLock); + } int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); @@ -342,9 +344,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) { sdbFreeRow(pSdb, pRow, true); } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } } +void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); } + void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { *ppObj = NULL; @@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { return ppRow; } -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) { +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) { *ppObj = NULL; SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + if (lock) { + taosThreadRwlockRdlock(pLock); + } SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { @@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat *status = pRow->status; break; } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } return ppRow; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c8a4ff945d..97b0be07c1 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -693,6 +693,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; + ASSERT(pAlterTbReq->colName); if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 59a9f32642..e1578b37f1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -171,11 +171,11 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); - } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); - } + /*if (pRsp->blockNum > 0) {*/ + /*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/ + /*} else {*/ + ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + /*}*/ } int32_t len = 0; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index bcdac1941a..4083a1a0ae 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -245,7 +245,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tqDebug("vgId:%d cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey); continue; } - if (pPushEntry->dataRsp.reqOffset.version > ver) { + if (pPushEntry->dataRsp.reqOffset.version >= ver) { tqDebug("vgId:%d push entry req version %ld, while push version %ld, skip", pTq->pVnode->config.vgId, pPushEntry->dataRsp.reqOffset.version, ver); continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 50c7a7cc53..0ee2e31097 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD if (code) goto _err; } +_err: if (aDelData) { taosArrayDestroy(aDelData); } -_err: return code; } @@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); - if (code) goto _err; *pDelIdx = *pIdx; +_err: if (pDelIdxArray) { taosArrayDestroy(pDelIdxArray); } -_err: return code; } @@ -599,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (code) goto _err; - if (!state->pBlockIdx) { goto _next_fileset; } @@ -898,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs if (code) goto _err; code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } tsdbDelFReaderClose(&pDelFReader); } else { @@ -1231,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { _err: nextRowIterClose(&iter); taosMemoryFreeClear(pTSchema); + *ppLastArray = NULL; + taosArrayDestroy(pColArray); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3db0b4f2b9..a4d993bc6c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -396,12 +396,19 @@ _exit: return code; } +static int32_t tDataIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + SDataIter *pIter1 = (SDataIter *)((uint8_t *)n1 - offsetof(SDataIter, n)); + SDataIter *pIter2 = (SDataIter *)((uint8_t *)n2 - offsetof(SDataIter, n)); + + return tRowInfoCmprFn(&pIter1->r, &pIter2->r); +} + static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; int32_t lino = 0; pCommitter->pIter = NULL; - tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pCommitter->rbt, tDataIterCmprFn); // memory TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 85514ed5b6..d08b743763 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -610,9 +610,6 @@ int32_t tsdbFSRollback(STsdbFS *pFS) { ASSERT(0); return code; - -_err: - return code; } int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) { @@ -866,7 +863,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); if (nRef == 0) { tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pSmaF); } } else { @@ -877,7 +874,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { // stt if (sameDisk) { if (pSetNew->nSttF > pSetOld->nSttF) { - ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + ASSERT(pSetNew->nSttF == pSetOld->nSttF + 1); pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -964,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSetOld->pHeadF); } @@ -1104,7 +1101,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbDelFileName(pTsdb, pFS->pDelFile, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pFS->pDelFile); } } @@ -1117,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSet->pHeadF); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 6679abd247..0189ced3c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -562,6 +562,8 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i // backward put first data row.pTSRow = tGetSubmitBlkNext(&blkIter); + if (row.pTSRow == NULL) return code; + key.ts = row.pTSRow->ts; nRow++; tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 0b7d1cbaf4..09a46b59bb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -504,9 +504,9 @@ _exit: SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } // SMergeTree ================================================= -static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { - SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode)); - SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) { + SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node)); + SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node)); TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); @@ -543,6 +543,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; + ASSERT(pMTree->pLoadInfo != NULL); for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter *pIter = NULL; @@ -583,7 +584,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node)); + int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index e4080ccf1e..5197823490 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -87,9 +87,13 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { - taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + taosThreadRwlockWrlock(&(*pTsdb)->rwLock); tsdbMemTableDestroy((*pTsdb)->mem); (*pTsdb)->mem = NULL; + taosThreadRwlockUnlock(&(*pTsdb)->rwLock); + + taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + tsdbFSClose(*pTsdb); tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a7850f3114..6b706901ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3395,19 +3395,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - code = doOpenReaderImpl(pReader); + if (numOfTables > 0) { + code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { - return code; + goto _err; } - } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - STsdbReader* pNextReader = pReader->innerReader[1]; + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; // we need only one row pPrevReader->capacity = 1; @@ -3422,19 +3423,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl pNextReader->pMemSchema = pReader->pMemSchema; pNextReader->pReadSnap = pReader->pReadSnap; - code = doOpenReaderImpl(pPrevReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pNextReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pNextReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } @@ -3513,6 +3515,10 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pLReader); } + if (pReader->innerReader[0] != 0) { + tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr); + } + tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 " SMA-time:%.2f ms, fileBlocks:%" PRId64 diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 167b4a104c..84b3afe72c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -18,7 +18,7 @@ // =============== PAGE-WISE FILE =============== static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; - STsdbFD *pFD; + STsdbFD *pFD = NULL; *ppFD = NULL; @@ -35,6 +35,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pFD = taosOpenFile(path, flag); if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD); goto _exit; } pFD->szPage = szPage; @@ -42,11 +43,15 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFD->pFD); taosMemoryFree(pFD); goto _exit; } if (taosStatFile(path, &pFD->szFile, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); goto _exit; } ASSERT(pFD->szFile % szPage == 0); @@ -59,10 +64,12 @@ _exit: static void tsdbCloseFile(STsdbFD **ppFD) { STsdbFD *pFD = *ppFD; - taosMemoryFree(pFD->pBuf); - taosCloseFile(&pFD->pFD); - taosMemoryFree(pFD); - *ppFD = NULL; + if (pFD) { + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); + *ppFD = NULL; + } } static int32_t tsdbWriteFilePage(STsdbFD *pFD) { @@ -117,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { } // check - if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } @@ -443,7 +450,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p pBlockIdx->size = size; pHeadFile->size += size; - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 " size:%" PRId64 " nItem:%d", TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); @@ -457,7 +464,7 @@ _err: int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { int32_t code = 0; SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; - int64_t size; + int64_t size = 0; int64_t n; // check @@ -836,7 +843,8 @@ _err: // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; - SDataFReader *pReader; + int32_t lino = 0; + SDataFReader *pReader = NULL; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fname[TSDB_FILENAME_LEN]; @@ -844,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->pSet = pSet; @@ -852,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // head tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // data tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // sma tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // stt for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppReader = pReader; - return code; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); -_err: - tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; + if (pReader) { + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]); + tsdbCloseFile(&pReader->pSmaFD); + tsdbCloseFile(&pReader->pDataFD); + tsdbCloseFile(&pReader->pHeadFD); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + } return code; } @@ -906,10 +923,6 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { taosMemoryFree(*ppReader); *ppReader = NULL; return code; - -_err: - tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code)); - return code; } int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { @@ -1289,16 +1302,17 @@ _exit: // SDelFWriter ==================================================== int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter; + SDelFWriter *pDelFWriter = NULL; int64_t n; // alloc pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); if (pDelFWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pDelFWriter->pTsdb = pTsdb; pDelFWriter->fDel = *pFile; @@ -1306,21 +1320,28 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // update header code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pDelFWriter->fDel.size = TSDB_FHDR_SIZE; pDelFWriter->fDel.offset = 0; *ppWriter = pDelFWriter; - return code; -_err: - tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; +_exit: + if (code) { + if (pDelFWriter) { + taosMemoryFree(pDelFWriter); + tsdbCloseFile(&pDelFWriter->pWriteH); + } + *ppWriter = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); + } else { + *ppWriter = pDelFWriter; + } return code; } @@ -1456,15 +1477,15 @@ struct SDelFReader { int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; - SDelFReader *pDelFReader; - int64_t n; + SDelFReader *pDelFReader = NULL; // alloc pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); if (pDelFReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } // open impl @@ -1473,14 +1494,18 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH); - if (code) goto _err; + if (code) { + taosMemoryFree(pDelFReader); + goto _exit; + } - *ppReader = pDelFReader; - return code; - -_err: - tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + *ppReader = pDelFReader; + } return code; } @@ -1573,4 +1598,4 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a928dc3484..74c704598d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -67,6 +67,13 @@ extern int32_t tRowInfoCmprFn(const void* p1, const void* p2); extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData); extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); +static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) { + SFDataIter* pIter1 = (SFDataIter*)(((uint8_t*)pNode1) - offsetof(SFDataIter, n)); + SFDataIter* pIter2 = (SFDataIter*)(((uint8_t*)pNode2) - offsetof(SFDataIter, n)); + + return tRowInfoCmprFn(&pIter1->rInfo, &pIter2->rInfo); +} + static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { int32_t code = 0; @@ -79,7 +86,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (code) goto _err; pReader->pIter = NULL; - tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn); // .data file SFDataIter* pIter = &pReader->aFDataIter[0]; @@ -421,7 +428,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { n += tPutDelData((*ppData) + n, pDelData); } - tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%d" PRId64 " size:%d", + tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%" PRId64 " size:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pDelIdx->suid, pDelIdx->uid, size); break; @@ -431,20 +438,21 @@ _exit: return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->pVnode, + tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); return code; } int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { int32_t code = 0; + int32_t lino = 0; STsdbSnapReader* pReader = NULL; // alloc pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->sver = sver; @@ -454,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = taosThreadRwlockRdlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFSRef(pTsdb, &pReader->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = taosThreadRwlockUnlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // data @@ -478,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pIter->aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } else { pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tBlockDataCreate(&pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pReader->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // del pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); if (pReader->aDelIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pReader->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); - *ppReader = pReader; - return code; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code), + pTsdb->path); + *ppReader = NULL; -_err: - tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppReader = NULL; + if (pReader) { + taosArrayDestroy(pReader->aDelData); + taosArrayDestroy(pReader->aDelIdx); + tBlockDataDestroy(&pReader->bData, 1); + tsdbFSDestroy(&pReader->fs); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); + } return code; } @@ -1247,20 +1264,21 @@ _err: // APIs int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { int32_t code = 0; + int32_t lino = 0; STsdbSnapWriter* pWriter = NULL; // alloc pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->pTsdb = pTsdb; pWriter->sver = sver; pWriter->ever = ever; code = tsdbFSCopy(pTsdb, &pWriter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // config pWriter->minutes = pTsdb->keepCfg.days; @@ -1272,7 +1290,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr // SNAP_DATA_TSDB code = tBlockDataCreate(&pWriter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pWriter->fid = INT32_MIN; pWriter->id = (TABLEID){0}; @@ -1280,53 +1298,67 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dReader.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // Writer pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pWriter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dWriter.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataCreate(&pWriter->dWriter.sData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // SNAP_DATA_DEL pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pWriter->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppWriter = pWriter; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + *ppWriter = NULL; - tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppWriter = NULL; + if (pWriter) { + if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW); + if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData); + if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR); + tBlockDataDestroy(&pWriter->dWriter.sData, 1); + tBlockDataDestroy(&pWriter->dWriter.bData, 1); + if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk); + if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx); + tBlockDataDestroy(&pWriter->dReader.bData, 1); + if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx); + tBlockDataDestroy(&pWriter->bData, 1); + tsdbFSDestroy(&pWriter->fs); + taosMemoryFree(pWriter); + } + } else { + tsdbDebug("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); + *ppWriter = pWriter; + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 0a9fbf92a4..f38802aee7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -34,7 +34,9 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - tInitSubmitMsgIter(pMsg, &msgIter); + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return -1; + } while (true) { SSubmitBlkRsp r = {0}; tGetSubmitMsgNext(&msgIter, &pBlock); diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index d21cd957a9..682f2b4225 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -143,17 +143,17 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { } void vnodeBufPoolFree(SVBufPool *pPool, void *p) { - uint8_t *ptr = (uint8_t *)p; - SVBufPoolNode *pNode; + // uint8_t *ptr = (uint8_t *)p; + // SVBufPoolNode *pNode; - if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { - pNode = &((SVBufPoolNode *)p)[-1]; - *pNode->pnext = pNode->prev; - pNode->prev->pnext = pNode->pnext; + // if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { + // pNode = &((SVBufPoolNode *)p)[-1]; + // *pNode->pnext = pNode->prev; + // pNode->prev->pnext = pNode->pnext; - pPool->size = pPool->size - sizeof(*pNode) - pNode->size; - taosMemoryFree(pNode); - } + // pPool->size = pPool->size - sizeof(*pNode) - pNode->size; + // taosMemoryFree(pNode); + // } } void vnodeBufPoolRef(SVBufPool *pPool) { diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 314c0e7390..9fe37505ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -46,11 +46,17 @@ int vnodeInit(int nthreads) { return 0; } - vnodeGlobal.stop = 0; + taosThreadMutexInit(&vnodeGlobal.mutex, NULL); + taosThreadCondInit(&vnodeGlobal.hasTask, NULL); + taosThreadMutexLock(&vnodeGlobal.mutex); + + vnodeGlobal.stop = 0; vnodeGlobal.queue.next = &vnodeGlobal.queue; vnodeGlobal.queue.prev = &vnodeGlobal.queue; + taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + vnodeGlobal.nthreads = nthreads; vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); if (vnodeGlobal.threads == NULL) { @@ -59,9 +65,6 @@ int vnodeInit(int nthreads) { return -1; } - taosThreadMutexInit(&vnodeGlobal.mutex, NULL); - taosThreadCondInit(&vnodeGlobal.hasTask, NULL); - for (int i = 0; i < nthreads; i++) { taosThreadCreate(&(vnodeGlobal.threads[i]), NULL, loop, NULL); } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 54ecba6d49..001bb5f7f2 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -38,7 +38,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { if (taosMkDir(path)) { return TAOS_SYSTEM_ERROR(errno); } - strcpy(dir, path); + snprintf(dir, TSDB_FILENAME_LEN, "%s", path); } if (pCfg) { @@ -51,7 +51,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.commitID = 0; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { - vError("vgId:%d, failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 08c3a34699..9cbb1b8af4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -166,7 +166,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (*ppData) { goto _exit; } else { - pReader->tqHandleDone = 1; + pReader->tqOffsetDone = 1; code = tqOffsetReaderClose(&pReader->pTqOffsetReader); if (code) goto _err; } @@ -219,7 +219,7 @@ _exit: return code; _err: - vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); return code; } @@ -260,7 +260,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr // commit it code = vnodeCommit(pVnode); - if (code) goto _err; + if (code) { + taosMemoryFree(pWriter); + goto _err; + } // inc commit ID pVnode->state.commitID++; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5f96fd8807..0f93e650c6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -43,14 +43,23 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); tStartDecode(&dc); - tDecodeI32v(&dc, &nReqs); + if (tDecodeI32v(&dc, &nReqs) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } for (int32_t iReq = 0; iReq < nReqs; iReq++) { tb_uid_t uid = tGenIdPI64(); char *name = NULL; tStartDecode(&dc); - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; @@ -68,7 +77,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int64_t ctime = taosGetTimestampMs(); tb_uid_t uid; - tInitSubmitMsgIter(pSubmitReq, &msgIter); + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + code = terrno; + goto _err; + } for (;;) { tGetSubmitMsgNext(&msgIter, &pBlock); @@ -78,10 +90,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { char *name = NULL; tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } uid = metaGetTableEntryUidByName(pVnode->pMeta, name); if (uid == 0) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 7b8c66e368..744d6eab06 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -638,7 +638,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_QNODE_LIST; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -692,7 +692,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_DNODE_LIST; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get dnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -743,7 +743,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU int32_t msgLen = 0; int32_t reqType = TDMT_MND_USE_DB; SCtgTask* pTask = tReq ? tReq->pTask : NULL; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -795,7 +795,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_DB_CFG; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); @@ -850,7 +850,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_INDEX; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get index from mnode, indexName:%s", indexName); @@ -905,7 +905,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(name, tbFName); @@ -962,7 +962,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_RETRIEVE_FUNC; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get udf info from mnode, funcName:%s", funcName); @@ -1017,7 +1017,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_USER_AUTH; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get user auth from mnode, user:%s", user); @@ -1077,7 +1077,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* int32_t reqType = TDMT_MND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFName, "%s.%s", dbFName, tbName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); @@ -1140,7 +1140,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa int32_t reqType = TDMT_VND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; ctgDebug("try to get table meta from vnode, vgId:%d, ep num:%d, ep %s:%d, tbFName:%s", vgroupInfo->vgId, @@ -1209,7 +1209,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t reqType = TDMT_VND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1274,7 +1274,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t reqType = TDMT_MND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1326,7 +1326,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_SERVER_VERSION; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; qDebug("try to get svr ver from mnode"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f9e47f92fe..fc996a6003 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -212,6 +212,7 @@ typedef struct SExprSupp { int32_t numOfExprs; // the number of scalar expression in group operator SqlFunctionCtx* pCtx; int32_t* rowEntryInfoOffset; // offset value for each row result cell info + SFilterInfo* pFilterInfo; } SExprSupp; typedef struct SOperatorInfo { @@ -479,6 +480,7 @@ typedef struct SStreamScanInfo { SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; int32_t primaryTsIndex; // primary time stamp slot id SReadHandle readHandle; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. @@ -775,6 +777,7 @@ typedef struct SStreamPartitionOperatorInfo { SPartitionBySupporter partitionSup; SExprSupp scalarSup; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; SHashObj* pPartitions; void* parIte; SSDataBlock* pInputDataBlock; @@ -924,7 +927,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2f6b737f79..0fadacb2c2 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1250,6 +1250,17 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pOpNode->node.aliasName); pExp->pExpr->_optrRoot.pRootNode = pNode; + } else if (type == QUERY_NODE_CASE_WHEN) { + pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; + SCaseWhenNode* pCaseNode = (SCaseWhenNode*)pNode; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pCaseNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, + pType->precision, pCaseNode->node.aliasName); + pExp->pExpr->_optrRoot.pRootNode = pNode; } else { ASSERT(0); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8c1104f519..ea01b961b1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1113,15 +1113,24 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo) { if (pFilterNode == NULL || pBlock->info.rows == 0) { return; } - SFilterInfo* filter = NULL; + SFilterInfo* filter = pFilterInfo; + int64_t st = taosGetTimestampUs(); + +// pError("start filter"); // todo move to the initialization function - int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + int32_t code = 0; + bool needFree = false; + if (filter == NULL) { + needFree = true; + code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + } + SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); @@ -1130,7 +1139,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM // todo the keep seems never to be True?? bool keep = filterExecute(filter, pBlock, &p, NULL, param1.numOfCols, &status); - filterFreeInfo(filter); + + if (needFree) { + filterFreeInfo(filter); + } extractQualifiedTupleByFilterResult(pBlock, p, keep, status); @@ -2479,7 +2491,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pAggInfo->pCondition, pInfo->pRes, NULL); + doFilter(pAggInfo->pCondition, pInfo->pRes, NULL, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -2873,7 +2885,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { break; } - doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo, NULL); if (fillResult->info.rows > 0) { break; } @@ -3049,6 +3061,12 @@ void cleanupExprSupp(SExprSupp* pSupp) { destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); taosMemoryFreeClear(pSupp->pExprInfo); } + + if (pSupp->pFilterInfo != NULL) { + filterFreeInfo(pSupp->pFilterInfo); + pSupp->pFilterInfo = NULL; + } + taosMemoryFree(pSupp->rowEntryInfoOffset); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f3ae652987..e62f72c8f8 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -311,7 +311,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes, NULL); + doFilter(pInfo->pCondition, pRes, NULL, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -991,6 +991,8 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFree(pInfo->partitionSup.keyBuf); cleanupExprSupp(&pInfo->scalarSup); + cleanupExprSupp(&pInfo->tbnameCalSup); + cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); taosMemoryFreeClear(param); } @@ -1037,6 +1039,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + if (pPartNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + int32_t keyLen = 0; code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index b4aa6de0a8..53cfa6c27a 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -387,7 +387,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { break; } if (pJoinInfo->pCondAfterMerge != NULL) { - doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL); + doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL, NULL); } if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index e9e6fed66a..fe6ee4129a 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -315,7 +315,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } // do apply filter - doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL); + doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL, NULL); // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { @@ -325,7 +325,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } else { // do apply filter if (pRes->info.rows > 0) { - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); if (pRes->info.rows == 0) { continue; } @@ -518,7 +518,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { } } - doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); + doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL, NULL); size_t rows = pInfo->pRes->info.rows; if (rows > 0 || pOperator->status == OP_EXEC_DONE) { break; @@ -620,7 +620,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { } pRes->info.rows = 1; - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); /*int32_t status = */ doIngroupLimitOffset(&pProjectInfo->limitInfo, 0, pRes, pOperator); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c20bb95811..c0c38e2b30 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -385,7 +385,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampUs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, pOperator->exprSupp.pFilterInfo); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -754,6 +754,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; + + if (pInfo->pFilterNode != NULL) { + code = filterInitFromNode((SNode*)pInfo->pFilterNode, &pOperator->exprSupp.pFilterInfo, 0); + } + pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; pInfo->currentGroupId = -1; @@ -1123,7 +1128,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 return NULL; } - doFilter(pInfo->pCondition, pResult, NULL); + doFilter(pInfo->pCondition, pResult, NULL, NULL); if (pResult->info.rows == 0) { continue; } @@ -1311,10 +1316,15 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL); + projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, pTagCalSup->numOfExprs, NULL); ASSERT(pResBlock->info.rows == 1); // build tagArray + /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/ + /*STagVal tagVal = {*/ + /*.cid = 0,*/ + /*.type = 0,*/ + /*};*/ // build STag // set STag @@ -1469,7 +1479,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } if (filter) { - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); } blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); @@ -1891,7 +1901,7 @@ FETCH_NEXT_BLOCK: } } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { @@ -2110,6 +2120,9 @@ static void destroyStreamScanOperatorInfo(void* param) { taosMemoryFree(pStreamScan->pPseudoExpr); } + cleanupExprSupp(&pStreamScan->tbnameCalSup); + cleanupExprSupp(&pStreamScan->tagCalSup); + updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); @@ -2164,6 +2177,19 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } + if (pTableScanNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2361,7 +2387,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } @@ -3434,7 +3460,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, NULL); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index ed3247e10e..c651529c7a 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { return NULL; } - doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo); + doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo, NULL); if (blockDataGetNumOfRows(pBlock) == 0) { continue; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 5be9918185..88af03775c 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1501,7 +1501,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } doStreamFillImpl(pOperator); - doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo, NULL); pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; if (pInfo->pRes->info.rows > 0) { break; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index df16914cb8..c1478bc935 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1277,7 +1277,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -1315,7 +1315,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBlock, NULL); + doFilter(pInfo->pCondition, pBlock, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -2019,7 +2019,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -2062,7 +2062,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -5251,7 +5251,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes); - doFilter(pMiaInfo->pCondition, pRes, NULL); + doFilter(pMiaInfo->pCondition, pRes, NULL, NULL); if (pRes->info.rows >= pOperator->resultInfo.capacity) { break; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index dac9b85740..b9b365fb42 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -131,7 +131,7 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { COPY_SCALAR_FIELD(placeholderNo); COPY_SCALAR_FIELD(typeData); COPY_SCALAR_FIELD(unit); - if (!pSrc->translate) { + if (!pSrc->translate || pSrc->isNull) { return TSDB_CODE_SUCCESS; } switch (pSrc->node.resType.type) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e401a3da7f..f0f1fc9975 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3011,7 +3011,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkValueIsNull, pNode->isNull); } - if (TSDB_CODE_SUCCESS == code && pNode->translate) { + if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = datumToJson(pNode, pJson); } @@ -3161,7 +3161,7 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkValueIsNull, &pNode->isNull); } - if (TSDB_CODE_SUCCESS == code && pNode->translate) { + if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = jsonToDatum(pJson, pNode); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index cdc4e66e42..9e89955ae5 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -478,7 +478,10 @@ static int32_t tlvDecodeValueEnum(STlvDecoder* pDecoder, void* pValue, int16_t l return code; } -static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) { +static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue, int32_t size) { + if (pTlv->len > size - 1) { + return TSDB_CODE_FAILED; + } memcpy(pValue, pTlv->value, pTlv->len); return TSDB_CODE_SUCCESS; } @@ -919,9 +922,14 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) { } break; } - case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_JSON: { + if (pTlv->len <= 0 || pTlv->len > TSDB_MAX_JSON_TAG_LEN) { + code = TSDB_CODE_FAILED; + break; + } code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p); break; + } case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: // todo @@ -1097,7 +1105,7 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); break; case FUNCTION_CODE_FUNCTION_NAME: - code = tlvDecodeCStr(pTlv, pNode->functionName); + code = tlvDecodeCStr(pTlv, pNode->functionName, sizeof(pNode->functionName)); break; case FUNCTION_CODE_FUNCTION_ID: code = tlvDecodeI32(pTlv, &pNode->funcId); @@ -1226,10 +1234,10 @@ static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI32(pTlv, &pNode->acctId); break; case NAME_CODE_DB_NAME: - code = tlvDecodeCStr(pTlv, pNode->dbname); + code = tlvDecodeCStr(pTlv, pNode->dbname, sizeof(pNode->dbname)); break; case NAME_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tname); + code = tlvDecodeCStr(pTlv, pNode->tname, sizeof(pNode->tname)); break; default: break; @@ -1538,7 +1546,7 @@ static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) { tlvForEach(pDecoder, pTlv, code) { switch (pTlv->type) { case EP_CODE_FQDN: - code = tlvDecodeCStr(pTlv, pNode->fqdn); + code = tlvDecodeCStr(pTlv, pNode->fqdn, sizeof(pNode->fqdn)); break; case EP_CODE_port: code = tlvDecodeU16(pTlv, &pNode->port); @@ -3207,7 +3215,7 @@ static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_QUERY_INSERT_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tableName); + code = tlvDecodeCStr(pTlv, pNode->tableName, sizeof(pNode->tableName)); break; case PHY_QUERY_INSERT_CODE_VG_ID: code = tlvDecodeI32(pTlv, &pNode->vgId); @@ -3284,10 +3292,10 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_DELETER_CODE_TABLE_FNAME: - code = tlvDecodeCStr(pTlv, pNode->tableFName); + code = tlvDecodeCStr(pTlv, pNode->tableFName, sizeof(pNode->tableFName)); break; case PHY_DELETER_CODE_TS_COL_NAME: - code = tlvDecodeCStr(pTlv, pNode->tsColName); + code = tlvDecodeCStr(pTlv, pNode->tsColName, sizeof(pNode->tsColName)); break; case PHY_DELETER_CODE_DELETE_TIME_RANGE: code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 6e964fb53a..bc0c4d42bb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -209,6 +209,10 @@ int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) { } SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + if (NULL == pAllocator) { + nodesError("allocator id %" PRIx64 " weak reference failed", allocatorId); + return -1; + } return pAllocator->self; } @@ -1604,7 +1608,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || - QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type); + QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type || QUERY_NODE_CASE_WHEN == type); } bool nodesIsUnaryOp(const SOperatorNode* pOp) { @@ -1716,9 +1720,10 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = 0; if ('\0' == pCol->tableAlias[0]) { - len = sprintf(name, "%s", pCol->colName); + len = snprintf(name, sizeof(name), "%s", pCol->colName); + } else { + len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } - len = sprintf(name, "%s.%s", pCol->tableAlias, pCol->colName); if (NULL == taosHashGet(pCxt->pColHash, name, len)) { pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); if (TSDB_CODE_SUCCESS == pCxt->errCode) { diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index 787abf287e..fb4b46aa35 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -175,8 +175,6 @@ _end: void taosCleanupKeywordsTable(); -SToken tscReplaceStrToken(char **str, SToken *token, const char *newToken); - SToken taosTokenDup(SToken *pToken, char *buf, int32_t len); #ifdef __cplusplus diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2e6f839368..15b939f14b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -518,7 +518,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa if (NULL != pDbName) { COPY_STRING_FORM_ID_TOKEN(realTable->table.dbName, pDbName); } else { - strcpy(realTable->table.dbName, pCxt->pQueryCxt->db); + snprintf(realTable->table.dbName, sizeof(realTable->table.dbName), "%s", pCxt->pQueryCxt->db); } if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) { COPY_STRING_FORM_ID_TOKEN(realTable->table.tableAlias, pTableAlias); @@ -599,7 +599,7 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { state->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == state->pCol) { nodesDestroyNode((SNode*)state); - CHECK_OUT_OF_MEM(state->pCol); + CHECK_OUT_OF_MEM(NULL); } state->pExpr = pExpr; return (SNode*)state; @@ -613,7 +613,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode interval->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == interval->pCol) { nodesDestroyNode((SNode*)interval); - CHECK_OUT_OF_MEM(interval->pCol); + CHECK_OUT_OF_MEM(NULL); } interval->pInterval = pInterval; interval->pOffset = pOffset; @@ -631,7 +631,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == fill->pWStartTs) { nodesDestroyNode((SNode*)fill); - CHECK_OUT_OF_MEM(fill->pWStartTs); + CHECK_OUT_OF_MEM(NULL); } strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart"); return (SNode*)fill; @@ -1495,10 +1495,10 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno CHECK_OUT_OF_MEM(pStmt); pStmt->indexType = type; pStmt->ignoreExists = ignoreExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->dbName, sizeof(pStmt->dbName), "%s", ((SRealTableNode*)pRealTable)->table.dbName); + snprintf(pStmt->tableName, sizeof(pStmt->tableName), "%s", ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pIndexName); nodesDestroyNode(pRealTable); pStmt->pCols = pCols; @@ -1524,8 +1524,8 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->ignoreNotExists = ignoreNotExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); nodesDestroyNode(pIndexName); return (SNode*)pStmt; } @@ -1818,7 +1818,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(pFunc); - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) { nodesDestroyNode((SNode*)pFunc); CHECK_OUT_OF_MEM(NULL); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index eec20f91e4..5b861ef79a 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -378,7 +378,9 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time } else if (pToken->type == TK_TODAY) { ts = taosGetTimestampToday(timePrec); } else if (pToken->type == TK_NK_INTEGER) { - toInteger(pToken->z, pToken->n, 10, &ts); + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } } else { // parse the RFC-3339/ISO-8601 timestamp format string if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -591,8 +593,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } return func(pMsgBuf, &iv, pSchema->bytes, param); } @@ -894,10 +894,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } - val->i64 = iv; break; } @@ -1173,7 +1170,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* NEXT_TOKEN(pCxt->pSql, sToken); SName sname; - createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); + CHECK_CODE(createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&sname, dbFName); strcpy(pCxt->sTableName, sname.tname); @@ -1605,7 +1602,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (!pCxt->pComCxt->needMultiParse) { continue; } else { - parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); + parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); break; } } @@ -1613,7 +1610,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } - parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); @@ -1650,7 +1647,7 @@ static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) { pCxt->pComCxt->needMultiParse = false; return TSDB_CODE_SUCCESS; } - parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); // merge according to vgId if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index e8c877bed2..997e680c99 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -// clang-format off + #include "parInsertData.h" #include "catalog.h" @@ -25,8 +25,8 @@ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; + TSKEY skey; + void* payloadAddr; int16_t index; } SBlockKeyTuple; @@ -194,8 +194,8 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { SEncoder coder = {0}; - char* pBuf; - int32_t len; + char* pBuf; + int32_t len; int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); @@ -211,19 +211,19 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) } } - pBuf= pBlocks->pData + pBlocks->size; + pBuf = pBlocks->pData + pBlocks->size; tEncoderInit(&coder, pBuf, len); - tEncodeSVCreateTbReq(&coder, pCreateTbReq); + int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); tEncoderClear(&coder); - pBlocks->size += len; pBlocks->createTbReqLen = len; - return TSDB_CODE_SUCCESS; + + return code; } -int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, +int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, + int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); @@ -272,12 +272,12 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); -// if (!pDataBlock->cloned) { - // free the refcount for metermeta - taosMemoryFreeClear(pDataBlock->pTableMeta); + // if (!pDataBlock->cloned) { + // free the refcount for metermeta + taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); -// } + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + // } taosMemoryFreeClear(pDataBlock); } @@ -687,16 +687,16 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p STableDataBlocks** p = taosHashIterate(pHashObj, NULL); STableDataBlocks* pOneTableBlock = *p; SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger *pBlkRowMerger = NULL; + SBlockRowMerger* pBlkRowMerger = NULL; while (pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = - getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); + pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId + int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), + TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, + pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); @@ -708,7 +708,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p // the maximum expanded size in byte when a row-wise data is converted to SDataRow format int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen; + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + + pOneTableBlock->createTbReqLen; if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); @@ -861,7 +862,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); if (pBlock->pTableMeta) { - void *pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); if (NULL == pNewMeta) { taosMemoryFreeClear(*pDst); return TSDB_CODE_OUT_OF_MEMORY; @@ -887,20 +888,18 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI } pBlock->vgId = vgId; - + if (pBlock->pTableMeta) { pBlock->pTableMeta->uid = uid; pBlock->pTableMeta->vgId = vgId; } - + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); return TSDB_CODE_SUCCESS; } -STableMeta *qGetTableMetaInDataBlock(void* pDataBlock) { - return ((STableDataBlocks*)pDataBlock)->pTableMeta; -} +STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } void qFreeStmtDataBlock(void* pDataBlock) { if (pDataBlock == NULL) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 345b9cc5b9..76c01cfd90 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -613,27 +613,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { return 0; } -SToken tscReplaceStrToken(char** str, SToken* token, const char* newToken) { - char* src = *str; - size_t nsize = strlen(newToken); - int32_t size = (int32_t)strlen(*str) - token->n + (int32_t)nsize + 1; - int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); - SToken ntoken; - - *str = taosMemoryCalloc(1, size); - - strncpy(*str, src, bsize); - strcat(*str, newToken); - strcat(*str, token->z + token->n); - - ntoken.n = (uint32_t)nsize; - ntoken.z = *str + bsize; - - taosMemoryFreeClear(src); - - return ntoken; -} - SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { SToken t0 = {0}; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b398b645b3..dbe0c97ab6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -292,9 +292,13 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { } else { do { SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (NULL == pTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (pCxt->currLevel == currTotalLevel) { taosArrayPush(pTables, &pTable); if (hasSameTableAlias(pTables)) { + taosArrayDestroy(pTables); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias); } @@ -308,7 +312,7 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { SFullDatabaseName name = {0}; - strcpy(name.fullDbName, pFullDbName); + snprintf(name.fullDbName, sizeof(name.fullDbName), "%s", pFullDbName); return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName)); } @@ -540,15 +544,17 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray if (TSDB_CODE_SUCCESS == code) { code = collectUseTable(pName, pCxt->pTables); } - if (pParCxt->async) { - code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); - } else { - SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, - .requestId = pParCxt->requestId, - .requestObjRefId = pParCxt->requestRid, - .mgmtEps = pParCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS == code) { + if (pParCxt->async) { + code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); + } else { + SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; - code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + } } if (TSDB_CODE_SUCCESS != code) { parserError("0x%" PRIx64 " getTableIndex error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, @@ -990,9 +996,9 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) return pCxt->errCode; } if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.u; + pVal->datum.i = (int64_t)pVal->datum.u; } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.d; + pVal->datum.i = (int64_t)pVal->datum.d; } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { pVal->datum.i = pVal->datum.b; } @@ -1069,7 +1075,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_BIGINT: { code = toInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.i); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_BIGINT(pVal->datum.i))) { + if (strict && TSDB_CODE_SUCCESS != code) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -1888,6 +1894,7 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW pWhenThen->pWhen = pIsTrue; } if (first) { + first = false; pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; } else if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { SNode* pCastFunc = NULL; @@ -3429,10 +3436,10 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return NULL; } pCol->node.resType = ((SExprNode*)pNode)->resType; - strcpy(pCol->tableAlias, pTableAlias); - strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); - strcpy(pCol->node.aliasName, pCol->colName); - strcpy(pCol->node.userAlias, ((SExprNode*)pNode)->userAlias); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pTableAlias); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", ((SExprNode*)pNode)->aliasName); + snprintf(pCol->node.aliasName, sizeof(pCol->node.aliasName), "%s", pCol->colName); + snprintf(pCol->node.userAlias, sizeof(pCol->node.userAlias), "%s", ((SExprNode*)pNode)->userAlias); return (SNode*)pCol; } @@ -3739,8 +3746,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int32_t val, int32_t minVal, int32_t maxVal) { if (val >= 0 && (val < minVal || val > maxVal)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 " valid range: [%d, %d]", pName, - val, minVal, maxVal); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %d valid range: [%d, %d]", pName, val, + minVal, maxVal); } return TSDB_CODE_SUCCESS; } @@ -3877,7 +3884,7 @@ static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pO static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) { if (val >= 0 && val != v1 && val != v2) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option %s: %" PRId64 ", only %d, %d allowed", pName, val, v1, v2); + "Invalid option %s: %d, only %d, %d allowed", pName, val, v1, v2); } return TSDB_CODE_SUCCESS; } @@ -4462,8 +4469,8 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pTable->table.dbName, pInfo->pDbName); - strcpy(pTable->table.tableName, pInfo->pTableName); + snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); + snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); pSelect->pFromTable = (SNode*)pTable; @@ -4939,9 +4946,9 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt alterReq.superUser = 0; alterReq.enable = pStmt->enable; alterReq.sysInfo = pStmt->sysinfo; - strcpy(alterReq.pass, pStmt->password); + snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password); if (NULL != pCxt->pParseCxt->db) { - strcpy(alterReq.dbname, pCxt->pParseCxt->db); + snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db); } return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq); @@ -6048,7 +6055,7 @@ static SNode* createProjectCol(const char* pProjCol) { if (NULL == pCol) { return NULL; } - strcpy(pCol->colName, pProjCol); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pProjCol); return (SNode*)pCol; } @@ -6081,9 +6088,9 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32 nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pRealTable->table.dbName, pDb); - strcpy(pRealTable->table.tableName, pTable); - strcpy(pRealTable->table.tableAlias, pTable); + snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); + snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); + snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); pSelect->pFromTable = (SNode*)pRealTable; if (numOfProjs >= 0) { @@ -7124,8 +7131,9 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p pReq->newComment = strdup(pStmt->pOptions->comment); if (NULL == pReq->newComment) { code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pReq->newCommentLen = strlen(pReq->newComment); } - pReq->newCommentLen = strlen(pReq->newComment); } else { pReq->newCommentLen = -1; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dab1f01574..ee82e6c04b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -381,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); + taosMemoryFree(tmp); goto end; } val.nData = valLen; @@ -652,8 +653,8 @@ static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCac } SUserAuthInfo auth = {0}; - strcpy(auth.user, pCxt->pUser); - strcpy(auth.dbFName, p->dbFName); + snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser); + snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName); auth.type = AUTH_TYPE_WRITE; taosArrayPush(pCatalogReq->pUser, &auth); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e6868f0ceb..9e1a01b13d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -66,7 +66,8 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { switch (nodeType(*pNode)) { case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { + case QUERY_NODE_FUNCTION: + case QUERY_NODE_CASE_WHEN: { SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; SNode* pExpr; int32_t index = 0; @@ -118,6 +119,17 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } +static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { + nodesWalkExpr(pExpr, doNameExpr, NULL); + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL}; + cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); + if (TSDB_CODE_SUCCESS == cxt.errCode) { + nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); + nodesClearList(cxt.pExprs); + } + return cxt.errCode; +} + static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; @@ -459,9 +471,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) return NULL; } pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pExpr->aliasName); if (NULL != pStmtName) { - strcpy(pCol->tableAlias, pStmtName); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName); } return pCol; } @@ -711,8 +723,13 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo nodesDestroyNode((SNode*)pWindow); return TSDB_CODE_OUT_OF_MEMORY; } - - return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); + // rewrite the expression in subsequent clauses + int32_t code = rewriteExprForSelect(pWindow->pStateExpr, pSelect, SQL_CLAUSE_WINDOW); + if (TSDB_CODE_SUCCESS == code) { + code = createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); + } + + return code; } static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 50f1f4b369..cd02e7a136 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1124,7 +1124,7 @@ static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, SNodeList** pS bool notOptimize = false; int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, ¬Optimize, pSequencingNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { - nodesClearList(*pSequencingNodes); + NODES_CLEAR_LIST(*pSequencingNodes); } return code; } @@ -1361,74 +1361,6 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo return code; } -static SNode* smaIndexOptCreateWStartTs() { - SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pWStart) { - return NULL; - } - strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pWStart, NULL, 0)) { - nodesDestroyNode((SNode*)pWStart); - return NULL; - } - return (SNode*)pWStart; -} - -static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { - SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pMergeKey) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->pExpr = nodesCloneNode(pCol); - if (NULL == pMergeKey->pExpr) { - nodesDestroyNode((SNode*)pMergeKey); - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->order = ORDER_ASC; - pMergeKey->nullOrder = NULL_ORDER_FIRST; - return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); -} - -static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { - if (wstrartIndex < 0) { - SNode* pWStart = smaIndexOptCreateWStartTs(); - if (NULL == pWStart) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t code = createColumnByRewriteExpr(pWStart, &pInterval->node.pTargets); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pWStart); - return code; - } - wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; - } - return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); -} - -static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { - SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; - SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); - if (NULL == pMergeTargets) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SLogicNode* pSmaScan = NULL; - SLogicNode* pMerge = NULL; - SNodeList* pMergeKeys = NULL; - int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); - } - return code; -} - static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pSmaCols, int32_t wstrartIndex) { SLogicNode* pSmaScan = NULL; @@ -1559,7 +1491,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) { return NULL; } - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { SColumnNode* pCol = (SColumnNode*)pNode; partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName); @@ -2028,7 +1960,7 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode if (NULL != pSelectValue) { strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); } else { - snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc); + snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc); } int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol)); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 2935c647aa..8ff8538c0d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -67,7 +67,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const S if (NULL == pSlot) { return NULL; } - strcpy(pSlot->name, pName); + snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -663,13 +663,17 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); } - SNodeList* condCols = nodesMakeList(); if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { - code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols); - } - if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc); - nodesDestroyList(condCols); + SNodeList* pCondCols = nodesMakeList(); + if (NULL == pCondCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &pCondCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pCondCols, pJoin->node.pOutputDataBlockDesc); + } + nodesDestroyList(pCondCols); } if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { @@ -761,7 +765,8 @@ static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: { + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_CASE_WHEN: { return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_FUNCTION: { @@ -1633,7 +1638,7 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; if (NULL != pCxt->pPlanCxt->pUser) { - strcpy(pSubplan->user, pCxt->pPlanCxt->pUser); + snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser); } return pSubplan; } @@ -1824,7 +1829,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNode* pSubplan, int32_t lev return TSDB_CODE_OUT_OF_MEMORY; } } - return nodesListStrictAppend(pGroup->pNodeList, (SNode*)pSubplan); + return nodesListAppend(pGroup->pNodeList, (SNode*)pSubplan); } static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 74ed3b57a4..9567a2388f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -342,7 +342,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart); int32_t code = fmGetFuncInfo(pWStart, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pFuncs, (SNode*)pWStart); @@ -367,7 +367,7 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWEnd->functionName, "_wend"); - snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, pWEnd); + snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd); int32_t code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd); @@ -389,22 +389,19 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic SNode* pConditions = pMergeWindow->node.pConditions; pMergeWindow->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SWindowLogicNode* pPartWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow); if (NULL == pPartWin) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartWin->node.groupAction = GROUP_ACTION_KEEP; + pMergeWindow->node.pTargets = pTargets; + pMergeWindow->node.pConditions = pConditions; + pPartWin->node.pChildren = pChildren; + splSetParent((SLogicNode*)pPartWin); - if (TSDB_CODE_SUCCESS == code) { - pMergeWindow->node.pTargets = pTargets; - pMergeWindow->node.pConditions = pConditions; - pPartWin->node.pChildren = pChildren; - splSetParent((SLogicNode*)pPartWin); - code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); - } int32_t index = 0; + int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = stbSplAppendWStart(pPartWin->pFuncs, &index); } @@ -721,15 +718,16 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO SNode* pConditions = pMergeAgg->node.pConditions; pMergeAgg->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SAggLogicNode* pPartAgg = (SAggLogicNode*)nodesCloneNode((SNode*)pMergeAgg); if (NULL == pPartAgg) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartAgg->node.groupAction = GROUP_ACTION_KEEP; - if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index ac1589bec0..a13e959a36 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -53,7 +53,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { + case QUERY_NODE_FUNCTION: + case QUERY_NODE_CASE_WHEN: { SExprNode* pExpr = (SExprNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 621a38ea4a..80ba356e8f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -24,8 +24,8 @@ #pragma GCC diagnostic ignored "-Wformat-truncation" #endif -int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallocFp)(int32_t)) = {0}; +int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallocFp)(int64_t)) = {0}; int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0}; int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { @@ -67,8 +67,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { return TSDB_CODE_SUCCESS; } -int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallcFp)(int32_t)) { +int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallcFp)(int64_t)) { SBuildTableInput *pInput = input; if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -91,7 +91,7 @@ int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { SBuildUseDBInput *pInput = input; if (NULL == pInput || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -114,7 +114,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms return TSDB_CODE_SUCCESS; } -int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -132,7 +132,7 @@ int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -150,7 +150,7 @@ int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -167,7 +167,7 @@ int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -185,7 +185,7 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -203,8 +203,8 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallcFp)(int32_t)) { +int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -227,7 +227,7 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -245,7 +245,7 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -263,7 +263,7 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 626e78b4d3..36c6817595 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -123,6 +123,7 @@ typedef struct SQWTaskCtx { int32_t execId; int32_t level; + bool queryGotData; bool queryRsped; bool queryEnd; bool queryContinue; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9115c2d3aa..d2c16a1cb5 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -10,6 +10,7 @@ #include "tdatablock.h" #include "tmsg.h" #include "tname.h" +#include "tglobal.h" SQWorkerMgmt gQwMgmt = { .lock = 0, @@ -92,6 +93,19 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { return TSDB_CODE_SUCCESS; } +int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) { + if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) { + if (!ctx->localExec) { + qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode)); + } + + ctx->queryRsped = true; + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; @@ -144,7 +158,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (numOfResBlock == 0) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); } else { - QW_TASK_DLOG("qExecTask done", ""); + QW_TASK_DLOG("qExecTask done, useconds:%" PRIu64, useconds); } dsEndPut(sinkHandle, useconds); @@ -234,6 +248,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t code = 0; SOutputData output = {0}; + if (NULL == ctx->sinkHandle) { + return TSDB_CODE_SUCCESS; + } + *dataLen = 0; while (true) { @@ -407,6 +425,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (!ctx->queryRsped) { QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); @@ -419,6 +442,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); @@ -499,21 +527,17 @@ _return: if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC); + ctx->queryGotData = true; } - if (QW_PHASE_POST_QUERY == phase && ctx) { - if (!ctx->localExec) { - bool rsped = false; - SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; - qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); - qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); - if (!rsped) { - qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); - } + if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) { + bool rsped = false; + SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + if (!rsped) { + qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false); } - - ctx->queryRsped = true; } if (ctx) { @@ -551,6 +575,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); ctx->ctrlConnInfo = qwMsg->connInfo; + ctx->phase = -1; QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); @@ -604,6 +629,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } + qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); @@ -619,6 +646,31 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); + if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + void *rsp = NULL; + int32_t dataLen = 0; + SOutputData sOutput = {0}; + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + + if (rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + } + + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, + tstrerror(code), dataLen); + } + } + QW_RET(TSDB_CODE_SUCCESS); } @@ -740,7 +792,9 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { locked = true; // RC WARNING - if (QW_QUERY_RUNNING(ctx)) { + if (-1 == ctx->phase || false == ctx->queryGotData) { + QW_TASK_DLOG_E("task query unfinished"); + } else if (QW_QUERY_RUNNING(ctx)) { atomic_store_8((int8_t *)&ctx->queryContinue, 1); } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 0a9913e004..d3f29c0e49 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -86,7 +86,7 @@ typedef struct SScalarCtx { } \ } while (0) -int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow); +int32_t sclConvertValueToSclParam(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow); int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); @@ -95,6 +95,11 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) void sclFreeParam(SScalarParam* param); +void doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr); +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr); +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr); #ifdef __cplusplus } diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 1a749c61b3..e633b39223 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -20,6 +20,15 @@ extern "C" { #endif +typedef struct SSclVectorConvCtx { + const SScalarParam* pIn; + SScalarParam* pOut; + int32_t startIndex; + int32_t endIndex; + int16_t inType; + int16_t outType; +} SSclVectorConvCtx; + typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); static FORCE_INLINE double getVectorDoubleValue_TINYINT(void *src, int32_t index) { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 132e0ed214..81ee173648 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1165,7 +1165,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { SValueNode *valueNode = (SValueNode *)cell->pNode; if (valueNode->node.resType.type != type) { int32_t overflow = 0; - code = doConvertDataType(valueNode, &out, &overflow); + code = sclConvertValueToSclParam(valueNode, &out, &overflow); if (code) { // fltError("convert from %d to %d failed", in.type, out.type); FLT_ERR_RET(code); @@ -1973,7 +1973,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } // todo refactor the convert - int32_t code = doConvertDataType(var, &out, NULL); + int32_t code = sclConvertValueToSclParam(var, &out, NULL); if (code != TSDB_CODE_SUCCESS) { qError("convert value to type[%d] failed", type); return TSDB_CODE_TSC_INVALID_OPERATION; @@ -3792,6 +3792,11 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { return DEAL_RES_CONTINUE; } + if (QUERY_NODE_CASE_WHEN == nodeType(*pNode) || QUERY_NODE_WHEN_THEN == nodeType(*pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { SOperatorNode *node = (SOperatorNode *)*pNode; if (!FLT_IS_COMPARISON_OPERATOR(node->opType)) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index da7e286036..1f09fd4799 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -61,7 +61,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -int32_t doConvertDataType(SValueNode *pValueNode, SScalarParam *out, int32_t *overflow) { +int32_t sclConvertValueToSclParam(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow) { SScalarParam in = {.numOfRows = 1}; int32_t code = sclCreateColumnInfoData(&pValueNode->node.resType, 1, &in); if (code != TSDB_CODE_SUCCESS) { @@ -71,12 +71,34 @@ int32_t doConvertDataType(SValueNode *pValueNode, SScalarParam *out, int32_t *ov colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colInfoDataEnsureCapacity(out->columnData, 1); - code = vectorConvertImpl(&in, out, overflow); + code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); sclFreeParam(&in); return code; } +int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { + SSDataBlock* pb = taosArrayGetP(pBlockList, 0); + SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); + if (NULL == pLeft) { + sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pLeft->numOfRows = pb->info.rows; + + if (pDst->numOfRows < pb->info.rows) { + colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows); + } + + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); + OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); + + taosMemoryFree(pLeft); + + return TSDB_CODE_SUCCESS; +} + int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { @@ -110,7 +132,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { } int32_t overflow = 0; - code = doConvertDataType(valueNode, &out, &overflow); + code = sclConvertValueToSclParam(valueNode, &out, &overflow); if (code != TSDB_CODE_SUCCESS) { // sclError("convert data from %d to %d failed", in.type, out.type); SCL_ERR_JRET(code); @@ -178,7 +200,7 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParam(SScalarParam *param) { - if (!param->colAlloced) { + if (NULL == param || !param->colAlloced) { return; } @@ -386,7 +408,8 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t } case QUERY_NODE_FUNCTION: case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: { + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_CASE_WHEN: { SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); @@ -540,6 +563,135 @@ _return: SCL_RET(code); } +int32_t sclGetNodeRes(SNode* node, SScalarCtx *ctx, SScalarParam **res) { + if (NULL == node) { + return TSDB_CODE_SUCCESS; + } + + int32_t rowNum = 0; + *res = taosMemoryCalloc(1, sizeof(**res)); + if (NULL == *res) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCL_ERR_RET(sclInitParam(node, *res, ctx, &rowNum)); + + return TSDB_CODE_SUCCESS; +} + +int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCell, SScalarParam *pCase, SScalarParam *pElse, SScalarParam *pComp, SScalarParam *output, int32_t rowIdx, int32_t totalRows, bool *complete) { + SNode *node = NULL; + SWhenThenNode* pWhenThen = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + int32_t code = 0; + + for (SListCell* cell = pCell; (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) { + pWhenThen = (SWhenThenNode*)node; + + SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL); + + bool *equal = (bool*)colDataGetData(pComp->columnData, rowIdx); + if (*equal) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0))); + + if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + } + + if (pElse) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0))); + + if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pElse->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + colDataAppend(output->columnData, rowIdx, NULL, true); + + if (0 == rowIdx && 1 == pCase->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + +_return: + + sclFreeParam(pWhen); + sclFreeParam(pThen); + + SCL_RET(code); +} + +int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCell, SScalarParam *pElse, SScalarParam *output, + int32_t rowIdx, int32_t totalRows, bool *complete, bool preSingle) { + SNode *node = NULL; + SWhenThenNode* pWhenThen = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + int32_t code = 0; + + for (SListCell* cell = pCell; (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) { + pWhenThen = (SWhenThenNode*)node; + pWhen = NULL; + pThen = NULL; + + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + bool *whenValue = (bool*)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? rowIdx : 0)); + + if (*whenValue) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0))); + + if (preSingle && 0 == rowIdx && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + sclFreeParam(pWhen); + sclFreeParam(pThen); + } + + if (pElse) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0))); + + if (preSingle && 0 == rowIdx && 1 == pElse->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + colDataAppend(output->columnData, rowIdx, NULL, true); + + if (preSingle && 0 == rowIdx && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + +_return: + + sclFreeParam(pWhen); + sclFreeParam(pThen); + + SCL_RET(code); +} + int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { SScalarParam *params = NULL; int32_t rowNum = 0; @@ -698,6 +850,101 @@ _return: SCL_RET(code); } +int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *output) { + int32_t code = 0; + SScalarParam *pCase = NULL; + SScalarParam *pElse = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + SScalarParam comp = {0}; + int32_t rowNum = 1; + bool complete = false; + + if (NULL == node->pWhenThenList || node->pWhenThenList->length <= 0) { + sclError("invalid whenThen list"); + SCL_ERR_RET(TSDB_CODE_INVALID_PARA); + } + + if (ctx->pBlockList) { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, 0); + rowNum = pb->info.rows; + output->numOfRows = pb->info.rows; + } + + SCL_ERR_JRET(sclCreateColumnInfoData(&node->node.resType, rowNum, output)); + + SCL_ERR_JRET(sclGetNodeRes(node->pCase, ctx, &pCase)); + SCL_ERR_JRET(sclGetNodeRes(node->pElse, ctx, &pElse)); + + SDataType compType = {0}; + compType.type = TSDB_DATA_TYPE_BOOL; + compType.bytes = tDataTypes[compType.type].bytes; + + SCL_ERR_JRET(sclCreateColumnInfoData(&compType, rowNum, &comp)); + + SNode* tnode = NULL; + SWhenThenNode* pWhenThen = (SWhenThenNode*)node->pWhenThenList->pHead->pNode; + + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + if (pCase) { + vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL); + + for (int32_t i = 0; i < rowNum; ++i) { + bool *equal = (bool*)colDataGetData(comp.columnData, (comp.numOfRows > 1 ? i : 0)); + if (*equal) { + colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); + if (0 == i && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + break; + } + } else { + SCL_ERR_JRET(sclWalkCaseWhenList(ctx, node->pWhenThenList, node->pWhenThenList->pHead->pNext, pCase, pElse, &comp, output, i, rowNum, &complete)); + if (complete) { + break; + } + } + } + } else { + for (int32_t i = 0; i < rowNum; ++i) { + bool *whenValue = (bool*)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? i : 0)); + if (*whenValue) { + colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); + if (0 == i && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + break; + } + } else { + SCL_ERR_JRET(sclWalkWhenList(ctx, node->pWhenThenList, node->pWhenThenList->pHead->pNext, pElse, output, i, rowNum, &complete, (pWhen->numOfRows == 1 && pThen->numOfRows == 1))); + if (complete) { + break; + } + } + } + } + + sclFreeParam(pCase); + sclFreeParam(pElse); + sclFreeParam(&comp); + sclFreeParam(pWhen); + sclFreeParam(pThen); + + return TSDB_CODE_SUCCESS; + +_return: + + sclFreeParam(pCase); + sclFreeParam(pElse); + sclFreeParam(&comp); + sclFreeParam(pWhen); + sclFreeParam(pThen); + sclFreeParam(output); + + SCL_RET(code); +} + + EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opType) { if (opType <= OP_TYPE_CALC_MAX) { SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); @@ -960,9 +1207,66 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } +EDealRes sclRewriteCaseWhen(SNode** pNode, SScalarCtx *ctx) { + SCaseWhenNode *node = (SCaseWhenNode *)*pNode; + + if ((!SCL_IS_CONST_NODE(node->pCase)) || (!SCL_IS_CONST_NODE(node->pElse))) { + return DEAL_RES_CONTINUE; + } + + SNode* tnode = NULL; + FOREACH(tnode, node->pWhenThenList) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)tnode; + if (!SCL_IS_CONST_NODE(pWhenThen->pWhen) || !SCL_IS_CONST_NODE(pWhenThen->pThen)) { + return DEAL_RES_CONTINUE; + } + } + + SScalarParam output = {0}; + ctx->code = sclExecCaseWhen(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->translate = true; + + res->node.resType = node->node.resType; + if (colDataIsNull_s(output.columnData, 0)) { + res->isNull = true; + res->node.resType = node->node.resType; + } else { + int32_t type = output.columnData->info.type; + if (IS_VAR_DATA_TYPE(type)) { // todo refactor + res->datum.p = output.columnData->pData; + output.columnData->pData = NULL; + } else { + nodesSetValueNodeValue(res, output.columnData->pData); + } + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + return DEAL_RES_CONTINUE; +} + + EDealRes sclConstantsRewriter(SNode **pNode, void *pContext) { SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + return sclRewriteOperator(pNode, ctx); + } + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { return sclRewriteFunction(pNode, ctx); } @@ -971,8 +1275,8 @@ EDealRes sclConstantsRewriter(SNode **pNode, void *pContext) { return sclRewriteLogic(pNode, ctx); } - if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { - return sclRewriteOperator(pNode, ctx); + if (QUERY_NODE_CASE_WHEN == nodeType(*pNode)) { + return sclRewriteCaseWhen(pNode, ctx); } return DEAL_RES_CONTINUE; @@ -1082,13 +1386,36 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } +EDealRes sclWalkCaseWhen(SNode* pNode, SScalarCtx *ctx) { + SCaseWhenNode *node = (SCaseWhenNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecCaseWhen(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + EDealRes sclCalcWalker(SNode *pNode, void *pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || - QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_LEFT_VALUE == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) + || QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_LEFT_VALUE == nodeType(pNode) + || QUERY_NODE_WHEN_THEN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + return sclWalkOperator(pNode, ctx); + } + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { return sclWalkFunction(pNode, ctx); } @@ -1097,38 +1424,19 @@ EDealRes sclCalcWalker(SNode *pNode, void *pContext) { return sclWalkLogic(pNode, ctx); } - if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - return sclWalkOperator(pNode, ctx); - } - if (QUERY_NODE_TARGET == nodeType(pNode)) { return sclWalkTarget(pNode, ctx); } + if (QUERY_NODE_CASE_WHEN == nodeType(pNode)) { + return sclWalkCaseWhen(pNode, ctx); + } + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } -int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { - SSDataBlock *pb = taosArrayGetP(pBlockList, 0); - SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); - if (NULL == pLeft) { - sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pLeft->numOfRows = pb->info.rows; - colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows); - - _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); - OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); - - taosMemoryFree(pLeft); - - return TSDB_CODE_SUCCESS; -} - int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) { if (NULL == pNode) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e83fe7ecd0..f467721248 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -367,46 +367,45 @@ static FORCE_INLINE void ncharToVar(char *buf, SScalarParam *pOut, int32_t rowIn taosMemoryFree(t); } -// TODO opt performance, tmp is not needed. -int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, int32_t inType, int32_t outType, - int32_t *overflow) { +//TODO opt performance, tmp is not needed. +int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t* overflow) { bool vton = false; _bufConverteFunc func = NULL; - if (TSDB_DATA_TYPE_BOOL == outType) { + if (TSDB_DATA_TYPE_BOOL == pCtx->outType) { func = varToBool; - } else if (IS_SIGNED_NUMERIC_TYPE(outType)) { + } else if (IS_SIGNED_NUMERIC_TYPE(pCtx->outType)) { func = varToSigned; - } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->outType)) { func = varToUnsigned; - } else if (IS_FLOAT_TYPE(outType)) { + } else if (IS_FLOAT_TYPE(pCtx->outType)) { func = varToFloat; - } else if (outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary - ASSERT(inType == TSDB_DATA_TYPE_NCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary + ASSERT(pCtx->inType == TSDB_DATA_TYPE_NCHAR); func = ncharToVar; vton = true; - } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar - ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar + ASSERT(pCtx->inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; vton = true; - } else if (TSDB_DATA_TYPE_TIMESTAMP == outType) { + } else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) { func = varToTimestamp; } else { - sclError("invalid convert outType:%d", outType); + sclError("invalid convert outType:%d", pCtx->outType); return TSDB_CODE_QRY_APP_ERROR; } - pOut->numOfRows = pIn->numOfRows; - for (int32_t i = 0; i < pIn->numOfRows; ++i) { - if (IS_HELPER_NULL(pIn->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); + pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { + if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) { + colDataAppendNULL(pCtx->pOut->columnData, i); continue; } - char *data = colDataGetVarData(pIn->columnData, i); - int32_t convertType = inType; - if (inType == TSDB_DATA_TYPE_JSON) { - if (*data == TSDB_DATA_TYPE_NULL) { + char* data = colDataGetVarData(pCtx->pIn->columnData, i); + int32_t convertType = pCtx->inType; + if(pCtx->inType == TSDB_DATA_TYPE_JSON){ + if(*data == TSDB_DATA_TYPE_NULL) { ASSERT(0); } else if (*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; @@ -415,13 +414,13 @@ int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, in terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; return terrno; } else { - convertNumberToNumber(data + CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType); + convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pCtx->pOut->columnData, i), *data, pCtx->outType); continue; } } - int32_t bufSize = pIn->columnData->info.bytes; - char *tmp = taosMemoryMalloc(varDataTLen(data)); - if (!tmp) { + int32_t bufSize = pCtx->pIn->columnData->info.bytes; + char *tmp = taosMemoryMalloc(varDataTLen(data)); + if(!tmp){ sclError("out of memory in vectorConvertFromVarData"); return TSDB_CODE_OUT_OF_MEMORY; } @@ -444,8 +443,8 @@ int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, in tmp[len] = 0; } } - - (*func)(tmp, pOut, i, overflow); + + (*func)(tmp, pCtx->pOut, i, overflow); taosMemoryFreeClear(tmp); } @@ -581,64 +580,64 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t return true; } -int32_t vectorConvertToVarData(const SScalarParam *pIn, SScalarParam *pOut, int16_t inType, int16_t outType) { - SColumnInfoData *pInputCol = pIn->columnData; - SColumnInfoData *pOutputCol = pOut->columnData; - char tmp[128] = {0}; +int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { + SColumnInfoData* pInputCol = pCtx->pIn->columnData; + SColumnInfoData* pOutputCol = pCtx->pOut->columnData; + char tmp[128] = {0}; - if (IS_SIGNED_NUMERIC_TYPE(inType) || inType == TSDB_DATA_TYPE_BOOL || inType == TSDB_DATA_TYPE_TIMESTAMP) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inType) || pCtx->inType == TSDB_DATA_TYPE_BOOL || pCtx->inType == TSDB_DATA_TYPE_TIMESTAMP) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } - } else if (IS_UNSIGNED_NUMERIC_TYPE(inType)) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inType)) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint64_t, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } - } else if (IS_FLOAT_TYPE(inType)) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + } else if (IS_FLOAT_TYPE(pCtx->inType)) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, double, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%lf", value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } } else { - sclError("not supported input type:%d", inType); + sclError("not supported input type:%d", pCtx->inType); return TSDB_CODE_QRY_APP_ERROR; } @@ -646,34 +645,35 @@ int32_t vectorConvertToVarData(const SScalarParam *pIn, SScalarParam *pOut, int1 } // TODO opt performance -int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow) { - SColumnInfoData *pInputCol = pIn->columnData; - SColumnInfoData *pOutputCol = pOut->columnData; +int32_t vectorConvertSingleColImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* overflow, int32_t startIndex, int32_t numOfRows) { + SColumnInfoData* pInputCol = pIn->columnData; + SColumnInfoData* pOutputCol = pOut->columnData; if (NULL == pInputCol) { sclError("input column is NULL, hashFilter %p", pIn->pHashFilter); return TSDB_CODE_APP_ERROR; } - int16_t inType = pInputCol->info.type; - int16_t outType = pOutputCol->info.type; + int32_t rstart = startIndex >= 0 ? startIndex : 0; + int32_t rend = numOfRows > 0 ? rstart + numOfRows - 1 : rstart + pIn->numOfRows - 1; + SSclVectorConvCtx cCtx = {pIn, pOut, rstart, rend, pInputCol->info.type, pOutputCol->info.type}; - if (IS_VAR_DATA_TYPE(inType)) { - return vectorConvertFromVarData(pIn, pOut, inType, outType, overflow); + if (IS_VAR_DATA_TYPE(cCtx.inType)) { + return vectorConvertFromVarData(&cCtx, overflow); } if (overflow) { ASSERT(1 == pIn->numOfRows); pOut->numOfRows = 0; - - if (IS_SIGNED_NUMERIC_TYPE(outType)) { - int64_t minValue = tDataTypes[outType].minValue; - int64_t maxValue = tDataTypes[outType].maxValue; - + + if (IS_SIGNED_NUMERIC_TYPE(cCtx.outType)) { + int64_t minValue = tDataTypes[cCtx.outType].minValue; + int64_t maxValue = tDataTypes[cCtx.outType].maxValue; + double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -683,13 +683,13 @@ int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t * } else { *overflow = 0; } - } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { - uint64_t minValue = (uint64_t)tDataTypes[outType].minValue; - uint64_t maxValue = (uint64_t)tDataTypes[outType].maxValue; - + } else if (IS_UNSIGNED_NUMERIC_TYPE(cCtx.outType)) { + uint64_t minValue = (uint64_t)tDataTypes[cCtx.outType].minValue; + uint64_t maxValue = (uint64_t)tDataTypes[cCtx.outType].maxValue; + double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -703,157 +703,157 @@ int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t * } pOut->numOfRows = pIn->numOfRows; - switch (outType) { + switch (cCtx.outType) { case TSDB_DATA_TYPE_BOOL: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } bool value = 0; - GET_TYPED_DATA(value, bool, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, bool, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } case TSDB_DATA_TYPE_TINYINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int8_t value = 0; - GET_TYPED_DATA(value, int8_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int8_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } - case TSDB_DATA_TYPE_SMALLINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_SMALLINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int16_t value = 0; - GET_TYPED_DATA(value, int16_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int16_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt16(pOutputCol, i, (int16_t *)&value); } break; } - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_INT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int32_t value = 0; - GET_TYPED_DATA(value, int32_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int32_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt32(pOutputCol, i, (int32_t *)&value); } break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt64(pOutputCol, i, (int64_t *)&value); } break; } - case TSDB_DATA_TYPE_UTINYINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_UTINYINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint8_t value = 0; - GET_TYPED_DATA(value, uint8_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint8_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } - case TSDB_DATA_TYPE_USMALLINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_USMALLINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint16_t value = 0; - GET_TYPED_DATA(value, uint16_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint16_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt16(pOutputCol, i, (int16_t *)&value); } break; } - case TSDB_DATA_TYPE_UINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_UINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint32_t value = 0; - GET_TYPED_DATA(value, uint32_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint32_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt32(pOutputCol, i, (int32_t *)&value); } break; } case TSDB_DATA_TYPE_UBIGINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); - colDataAppendInt64(pOutputCol, i, (int64_t *)&value); + GET_TYPED_DATA(value, uint64_t, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendInt64(pOutputCol, i, (int64_t*)&value); } break; } - case TSDB_DATA_TYPE_FLOAT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_FLOAT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } float value = 0; - GET_TYPED_DATA(value, float, inType, colDataGetData(pInputCol, i)); - colDataAppendFloat(pOutputCol, i, (float *)&value); + GET_TYPED_DATA(value, float, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendFloat(pOutputCol, i, (float*)&value); } break; } case TSDB_DATA_TYPE_DOUBLE: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); - colDataAppendDouble(pOutputCol, i, (double *)&value); + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendDouble(pOutputCol, i, (double*)&value); } break; } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { - return vectorConvertToVarData(pIn, pOut, inType, outType); + return vectorConvertToVarData(&cCtx); } default: - sclError("invalid convert output type:%d", outType); + sclError("invalid convert output type:%d", cCtx.outType); return TSDB_CODE_QRY_APP_ERROR; } @@ -894,7 +894,7 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { return gConvertTypes[type2][type1]; } -int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int32_t type) { +int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows) { SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; @@ -903,17 +903,16 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 return TSDB_CODE_OUT_OF_MEMORY; } - code = vectorConvertImpl(input, output, NULL); + code = vectorConvertSingleColImpl(input, output, NULL, startIndex, numOfRows); if (code) { - // taosMemoryFreeClear(paramOut1->data); return code; } return TSDB_CODE_SUCCESS; } -int32_t vectorConvert(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pLeftOut, SScalarParam *pRightOut) { - int32_t leftType = GET_PARAM_TYPE(pLeft); +int32_t vectorConvertCols(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut, int32_t startIndex, int32_t numOfRows) { + int32_t leftType = GET_PARAM_TYPE(pLeft); int32_t rightType = GET_PARAM_TYPE(pRight); if (leftType == rightType) { return TSDB_CODE_SUCCESS; @@ -941,14 +940,14 @@ int32_t vectorConvert(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p } if (type != GET_PARAM_TYPE(param1)) { - code = vectorConvertScalarParam(param1, paramOut1, type); + code = vectorConvertSingleCol(param1, paramOut1, type, startIndex, numOfRows); if (code) { return code; } } if (type != GET_PARAM_TYPE(param2)) { - code = vectorConvertScalarParam(param2, paramOut2, type); + code = vectorConvertSingleCol(param2, paramOut2, type, startIndex, numOfRows); if (code) { return code; } @@ -962,31 +961,6 @@ enum { VECTOR_UN_CONVERT = 0x2, }; -static int32_t doConvertHelper(SScalarParam *pDest, int32_t *convert, const SScalarParam *pParam, int32_t type) { - SColumnInfoData *pCol = pParam->columnData; - - if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { - pDest->numOfRows = pParam->numOfRows; - - SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - int32_t code = sclCreateColumnInfoData(&t, pParam->numOfRows, pDest); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = vectorConvertImpl(pParam, pDest, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - *convert = VECTOR_DO_CONVERT; - } else { - *convert = VECTOR_UN_CONVERT; - } - - return TSDB_CODE_SUCCESS; -} - // TODO not correct for descending order scan static void vectorMathAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { @@ -1030,20 +1004,25 @@ static void vectorMathTsAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pR } } -static SColumnInfoData *doVectorConvert(SScalarParam *pInput, int32_t *doConvert) { - SScalarParam convertParam = {0}; +static SColumnInfoData* vectorConvertVarToDouble(SScalarParam* pInput, int32_t* converted) { + SScalarParam output = {0}; + SColumnInfoData* pCol = pInput->columnData; - int32_t code = doConvertHelper(&convertParam, doConvert, pInput, TSDB_DATA_TYPE_DOUBLE); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; + if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { + int32_t code = vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, -1, -1); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + *converted = VECTOR_DO_CONVERT; + + return output.columnData; } - if (*doConvert == VECTOR_DO_CONVERT) { - return convertParam.columnData; - } else { - return pInput->columnData; - } + *converted = VECTOR_UN_CONVERT; + + return pInput->columnData; } static void doReleaseVec(SColumnInfoData *pCol, int32_t type) { @@ -1061,9 +1040,9 @@ void vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) || (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) || @@ -1168,9 +1147,9 @@ void vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BIGINT) || (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && @@ -1246,9 +1225,9 @@ void vectorMathMultiply(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1279,9 +1258,9 @@ void vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1333,9 +1312,9 @@ void vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1412,8 +1391,8 @@ void vectorMathMinus(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : (pLeft->numOfRows - 1); int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); + int32_t leftConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); @@ -1525,9 +1504,9 @@ void vectorBitAnd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); @@ -1579,9 +1558,9 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); @@ -1605,8 +1584,8 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, doReleaseVec(pRightCol, rightConvert); } -int32_t doVectorCompareImpl(int32_t numOfRows, SScalarParam *pOut, int32_t startIndex, int32_t step, __compar_fn_t fp, - SScalarParam *pLeft, SScalarParam *pRight, int32_t optr) { +int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t step, __compar_fn_t fp, int32_t optr) { int32_t num = 0; for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { @@ -1659,20 +1638,29 @@ int32_t doVectorCompareImpl(int32_t numOfRows, SScalarParam *pOut, int32_t start return num; } -void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; +void doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr) { + int32_t i = 0; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t lType = GET_PARAM_TYPE(pLeft); int32_t rType = GET_PARAM_TYPE(pRight); __compar_fn_t fp = NULL; - + int32_t compRows = 0; + if (lType == rType) { fp = filterGetCompFunc(lType, optr); } else { fp = filterGetCompFuncEx(lType, rType, optr); } - pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + if (startIndex < 0) { + i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + compRows = pOut->numOfRows; + } else { + compRows = startIndex + numOfRows; + i = startIndex; + } if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { @@ -1690,13 +1678,14 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam * } } } else { // normal compare - pOut->numOfQualified = doVectorCompareImpl(pOut->numOfRows, pOut, i, step, fp, pLeft, pRight, optr); + pOut->numOfQualified = doVectorCompareImpl(pLeft, pRight, pOut, i, compRows, step, fp, optr); } } -void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { - SScalarParam pLeftOut = {0}; - SScalarParam pRightOut = {0}; +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr) { + SScalarParam pLeftOut = {0}; + SScalarParam pRightOut = {0}; SScalarParam *param1 = NULL; SScalarParam *param2 = NULL; @@ -1704,7 +1693,7 @@ void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut param1 = pLeft; param2 = pRight; } else { - vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut); + vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows); if (pLeftOut.columnData != NULL) { param1 = &pLeftOut; @@ -1719,12 +1708,17 @@ void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut } } - vectorCompareImpl(param1, param2, pOut, _ord, optr); + doVectorCompare(param1, param2, pOut, startIndex, numOfRows, _ord, optr); + sclFreeParam(&pLeftOut); sclFreeParam(&pRightOut); } -void vectorGreater(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { + vectorCompareImpl(pLeft, pRight, pOut, -1, -1, _ord, optr); +} + +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_THAN); } @@ -1788,10 +1782,10 @@ void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut pOut->numOfRows = pLeft->numOfRows; } -void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { - vectorConvertImpl(pLeft, pOut, NULL); - for (int32_t i = 0; i < pOut->numOfRows; ++i) { - if (colDataIsNull_s(pOut->columnData, i)) { +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorConvertSingleColImpl(pLeft, pOut, NULL, -1, -1); + for(int32_t i = 0; i < pOut->numOfRows; ++i) { + if(colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; colDataAppendInt8(pOut->columnData, i, &v); colDataSetNotNull_f(pOut->columnData->nullbitmap, i); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 2678a652a9..33bf50d853 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -913,7 +913,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); +// SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); @@ -993,6 +993,12 @@ int32_t schLaunchTaskImpl(void *param) { SCH_ERR_JRET(schLaunchRemoteTask(pJob, pTask)); } +#if 0 + if (SCH_IS_QUERY_JOB(pJob)) { + SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask)); + } +#endif + _return: if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 813a11c056..c4b45649a4 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -155,8 +155,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { return; } - SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", errCode:0x%x", *jobId, errCode); - + SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); schHandleJobDrop(pJob, errCode); schReleaseJob(*jobId); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 4a63cd3bb2..ee317d0751 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -67,7 +67,11 @@ void streamSchedByTimer(void* param, void* tmrId) { atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - streamTaskInput(pTask, (SStreamQueueItem*)trigger); + if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) { + taosFreeQitem(trigger); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + return; + } streamSchedExec(pTask); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 63a28e17ed..53e49a6ba5 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -28,6 +28,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF sprintf(streamPath, "%s/%s", path, "stream"); pMeta->path = strdup(streamPath); if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db) < 0) { + taosMemoryFree(streamPath); goto _err; } @@ -58,7 +59,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF return pMeta; _err: - if (pMeta->path) taosMemoryFree(pMeta->path); + taosMemoryFree(pMeta->path); if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks); if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb); @@ -250,6 +251,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { + tdbFree(pKey); + tdbFree(pVal); return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); @@ -257,10 +260,14 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { tDecoderClear(&decoder); if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ea590c57e6..c0ed135d74 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -63,7 +63,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int sprintf(statePath, "%s/%d", path, pTask->taskId); } else { memset(statePath, 0, 300); - strncpy(statePath, path, 300); + tstrncpy(statePath, path, 300); } if (tdbOpen(statePath, szPage, pages, &pState->db) < 0) { goto _err; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 7c4aaba6ce..5aadb599a3 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -174,14 +174,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { if (res == TSDB_CODE_SUCCESS) { return false; } else { - qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 - ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } } - qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } @@ -203,14 +198,10 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { } if (ts < pInfo->minTS) { - qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } else if (res == TSDB_CODE_SUCCESS) { return false; } - qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, - maxTs, *pMapMaxTs, ts); // check from tsdb api return true; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0afc373f2d..101e99afea 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -169,7 +169,7 @@ typedef struct SSyncNode { } SSyncNode; // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index cc686ba2fe..907eb21f4c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -86,10 +86,10 @@ void syncCleanUp() { } } -int64_t syncOpen(const SSyncInfo* pSyncInfo) { +int64_t syncOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); if (pSyncNode == NULL) { - sError("failed to open sync node. vgId:%d", pSyncInfo->vgId); + sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr()); return -1; } @@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg return ret; } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { } ASSERT(rid == pSyncNode->rid); +#if 0 if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; @@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; +#else + syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); + syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return 0; +#endif } int32_t syncLeaderTransfer(int64_t rid) { @@ -919,9 +926,7 @@ _END: } // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { - SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; - +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode)); if (pSyncNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { sError("failed to open raft cfg file. path:%s", pSyncNode->configPath); goto _error; } - pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + if (pSyncInfo->syncCfg.replicaNum == 0) { + pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + } raftCfgClose(pSyncNode->pRaftCfg); pSyncNode->pRaftCfg = NULL; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 97e1b3f9cd..13b7e0ac0a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -984,7 +984,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch a new ofp and make it dirty SPgno pgno = 0; - SPage *ofp, *nextOfp; + SPage *ofp = NULL, *nextOfp = NULL; ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 593b8cc589..dd2416e5b8 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -34,9 +34,9 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); -static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) { - SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode)); - SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) { + SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node)); + SPage *pPageR = (SPage *)(((uint8_t *)rhs) - offsetof(SPage, node)); SPgno pgnoL = TDB_PAGE_PGNO(pPageL); SPgno pgnoR = TDB_PAGE_PGNO(pPageR); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index e5a783292d..c82af0d0e9 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -100,8 +100,8 @@ void rpcCloseImpl(void* arg) { taosMemoryFree(pRpc); } -void* rpcMallocCont(int32_t contLen) { - int32_t size = contLen + TRANS_MSG_OVERHEAD; +void* rpcMallocCont(int64_t contLen) { + int64_t size = contLen + TRANS_MSG_OVERHEAD; char* start = taosMemoryCalloc(1, size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); @@ -120,11 +120,11 @@ void rpcFreeCont(void* cont) { tTrace("rpc free cont:%p", (char*)cont - TRANS_MSG_OVERHEAD); } -void* rpcReallocCont(void* ptr, int32_t contLen) { +void* rpcReallocCont(void* ptr, int64_t contLen) { if (ptr == NULL) return rpcMallocCont(contLen); char* st = (char*)ptr - TRANS_MSG_OVERHEAD; - int32_t sz = contLen + TRANS_MSG_OVERHEAD; + int64_t sz = contLen + TRANS_MSG_OVERHEAD; st = taosMemoryRealloc(st, sz); if (st == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 83323725be..359b9824cf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -597,6 +597,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; + tDebug("%s conn %p alloc read buf", CONN_GET_INST_LABEL(conn), conn); transAllocBuffer(pBuf, buf); } static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { @@ -609,7 +610,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; while (transReadComplete(pBuf)) { - tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); + tDebug("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); if (pBuf->invalid) { cliHandleExcept(conn); break; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 42079ff005..f843c8559c 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -203,7 +203,7 @@ bool transReadComplete(SConnBuffer* connBuf) { } int transSetConnOption(uv_tcp_t* stream) { - uv_tcp_nodelay(stream, 1); + uv_tcp_nodelay(stream, 0); int ret = uv_tcp_keepalive(stream, 5, 60); return ret; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 57bf8a34b3..3616b4808f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -332,7 +332,10 @@ void uvOnSendCb(uv_write_t* req, int status) { if (status == 0) { tTrace("conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { - SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + STraceId* trace = &msg->msg.info.traceId; + tGDebug("conn %p write data out", conn); + destroySmsg(msg); // send cached data if (!transQueueEmpty(&conn->srvMsgs)) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 42291cd264..8808363bb2 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -855,6 +855,5 @@ int walRemoveMeta(SWal* pWal) { if (metaVer == -1) return 0; char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); - taosRemoveFile(fnameStr); - return 0; + return taosRemoveFile(fnameStr); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index cf96c0dae6..179d809c84 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -521,7 +521,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { pReader->pHead->head.version, ver); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -535,7 +534,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index e3045cb8e0..1e5c63fa8d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -42,10 +42,20 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, i); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); } } walRemoveMeta(pWal); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 8f026f6890..75a963d5c7 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -228,7 +228,7 @@ void taosPrintBackTrace() { void taosPrintBackTrace() { return; } #endif -void *taosMemoryMalloc(int32_t size) { +void *taosMemoryMalloc(int64_t size) { #ifdef USE_TD_MEMORY void *tmp = malloc(size + sizeof(TdMemoryInfo)); if (tmp == NULL) return NULL; @@ -244,7 +244,7 @@ void *taosMemoryMalloc(int32_t size) { #endif } -void *taosMemoryCalloc(int32_t num, int32_t size) { +void *taosMemoryCalloc(int64_t num, int64_t size) { #ifdef USE_TD_MEMORY int32_t memorySize = num * size; char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); @@ -261,7 +261,7 @@ void *taosMemoryCalloc(int32_t num, int32_t size) { #endif } -void *taosMemoryRealloc(void *ptr, int32_t size) { +void *taosMemoryRealloc(void *ptr, int64_t size) { #ifdef USE_TD_MEMORY if (ptr == NULL) return taosMemoryMalloc(size); @@ -318,7 +318,7 @@ void taosMemoryFree(void *ptr) { #endif } -int32_t taosMemorySize(void *ptr) { +int64_t taosMemorySize(void *ptr) { if (ptr == NULL) return 0; #ifdef USE_TD_MEMORY diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 65f1bac60a..cbbf53d8f1 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -219,7 +219,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { while (temp != pTree->NIL) { y = temp; - int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp)); + int32_t c = pTree->cmprFn(z, temp); if (c < 0) { temp = temp->left; } else if (c > 0) { @@ -232,7 +232,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { if (y == pTree->NIL) { pTree->root = z; - } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) { + } else if (pTree->cmprFn(z, y) < 0) { y->left = z; } else { y->right = z; @@ -245,10 +245,10 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { tRBTreePutFix(pTree, z); // update min/max node - if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) { + if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min, z) > 0) { pTree->min = z; } - if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) { + if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max, z) < 0) { pTree->max = z; } pTree->n++; @@ -309,11 +309,11 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { return pNode; } -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { - int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode)); + int32_t c = pTree->cmprFn(pKeyNode, pNode); if (c < 0) { pNode = pNode->left; diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 4ce668e6ba..d93e9fc569 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -145,7 +145,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it tSkipListWLock(pSkipList); void *pData = iterate(iter); - if (pData == NULL) return; + if (pData == NULL) { + tSkipListUnlock(pSkipList); + return; + } // backward to put the first data hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 0a835ec564..c68375894c 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -32,6 +32,10 @@ class TDTestCase: if "community" in selfPath: projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] else: projPath = selfPath[: selfPath.find("tests")] @@ -55,11 +59,10 @@ class TDTestCase: tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") - tdSql.query("select count(*) from (select distinct(tbname) from db.stb)") - tdSql.checkData(0, 0, 10) + tdSql.query("show db.tables") + tdSql.checkRows(10) tdSql.query("select count(*) from db.stb") - if len(tdSql.queryResult): - tdLog.exit("query result is %d" % len(tdSql.queryResult)) + tdSql.checkData(0, 0, 100) def stop(self): tdSql.close() diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json index d4b2aae2fb..da22ef75e2 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json @@ -20,8 +20,10 @@ "super_tables": [{ "name": "stb", "childtable_prefix": "stb_", + "childtable_count": 10, + "insert_rows": 10, "columns": [{"type": "INT"}], "tags": [{"type": "INT"}] }] }] -} \ No newline at end of file +} diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 82f73a4fdd..d8f4a36261 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -155,7 +155,7 @@ ./test.sh -f tsim/parser/select_with_tags.sim ./test.sh -f tsim/parser/selectResNum.sim ./test.sh -f tsim/parser/set_tag_vals.sim -./test.sh -f tsim/parser/single_row_in_tb.sim +# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim ./test.sh -f tsim/parser/sliding.sim ./test.sh -f tsim/parser/slimit_alter_tags.sim ./test.sh -f tsim/parser/slimit.sim @@ -194,7 +194,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -# TD-17919 ./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim # ---- show ---- @@ -352,6 +352,7 @@ ./test.sh -f tsim/scalar/in.sim ./test.sh -f tsim/scalar/scalar.sim ./test.sh -f tsim/scalar/filter.sim +./test.sh -f tsim/scalar/caseWhen.sim # ---- alter ---- ./test.sh -f tsim/alter/cached_schema_after_alter.sim diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim index 59156080c8..31fc276298 100644 --- a/tests/script/tsim/mnode/basic1.sim +++ b/tests/script/tsim/mnode/basic1.sim @@ -42,6 +42,7 @@ sql_error drop mnode on dnode 1 print =============== create mnode 2 sql create mnode on dnode 2 +print =============== create mnode 2 finished $x = 0 step2: $x = $x + 1 @@ -69,9 +70,10 @@ if $data(2)[2] != follower then goto step2 endi -sleep 2000 print ============ drop mnode 2 sql drop mnode on dnode 2 + +print ============ drop mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 1 then return -1 @@ -109,6 +111,8 @@ sleep 2000 print =============== create mnodes sql create mnode on dnode 2 + +print =============== create mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 2 then return -1 diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index a2b8aa2f96..6739a87b42 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -45,49 +45,12 @@ endi if $data(2)[4] != ready then goto step2 endi - -system sh/exec.sh -n dnode3 -s stop -sql_error create mnode on dnode 3 - -print =============== step3: select * from information_schema.ins_mnodes - -$x = 0 -step3: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi -sql select * from information_schema.ins_mnodes -x step3 -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] -print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] - -$leaderNum = 0 -if $data(1)[2] == leader then - $leaderNum = 1 -endi -if $data(2)[2] == leader then - $leaderNum = 1 -endi -if $leaderNum == 0 then - goto step3 -endi -if $data(3)[2] != offline then - goto step3 -endi -if $data(1)[3] != ready then - goto step3 -endi -if $data(2)[3] != ready then - goto step3 -endi -if $data(3)[3] != creating then - goto step3 +if $data(3)[4] != ready then + goto step2 endi -print =============== step4: start dnode3 -system sh/exec.sh -n dnode3 -s start +print =============== step4: create mnode 3 +sql create mnode on dnode 3 $x = 0 step4: @@ -159,12 +122,11 @@ endi if $data(2)[3] != ready then goto step5 endi -if $data(3)[3] != dropping then - goto step5 -endi print =============== step6: start dnode3 system sh/exec.sh -n dnode3 -s start +sql drop mnode on dnode 1 -x step60 +step60: $x = 0 step6: diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 387f38a717..16e8fa3dfa 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -232,12 +232,11 @@ endi if $leaderNum != 1 then goto step81 endi -if $data(1)[3] != dropping then - goto step81 -endi print =============== step9: start mnode1 and wait it dropped system sh/exec.sh -n dnode1 -s start +sql drop mnode on dnode 1 -x step90 +step90: $x = 0 step91: diff --git a/tests/script/tsim/scalar/caseWhen.sim b/tests/script/tsim/scalar/caseWhen.sim new file mode 100644 index 0000000000..f6b9c3ff08 --- /dev/null +++ b/tests/script/tsim/scalar/caseWhen.sim @@ -0,0 +1,1064 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== prepare data + +sql drop database if exists db1; +sql create database db1 vgroups 5; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(10), f3 bool) tags(t1 int, t2 bool, t3 binary(10)); +sql create table tba1 using sta tags(0, false, '0'); +sql create table tba2 using sta tags(1, true, '1'); +sql create table tba3 using sta tags(null, null, ''); +sql create table tba4 using sta tags(1, false, null); +sql create table tba5 using sta tags(3, true, 'aa'); +sql insert into tba1 values ('2022-09-26 15:15:01', 0, "a", false); +sql insert into tba1 values ('2022-09-26 15:15:02', 1, "0", true); +sql insert into tba1 values ('2022-09-26 15:15:03', 5, "5", false); +sql insert into tba1 values ('2022-09-26 15:15:04', null, null, null); +sql insert into tba2 values ('2022-09-27 15:15:01', 0, "a", false); +sql insert into tba2 values ('2022-09-27 15:15:02', 1, "0", true); +sql insert into tba2 values ('2022-09-27 15:15:03', 5, "5", false); +sql insert into tba2 values ('2022-09-27 15:15:04', null, null, null); +sql insert into tba3 values ('2022-09-28 15:15:01', 0, "a", false); +sql insert into tba3 values ('2022-09-28 15:15:02', 1, "0", true); +sql insert into tba3 values ('2022-09-28 15:15:03', 5, "5", false); +sql insert into tba3 values ('2022-09-28 15:15:04', null, null, null); +sql insert into tba4 values ('2022-09-29 15:15:01', 0, "a", false); +sql insert into tba4 values ('2022-09-29 15:15:02', 1, "0", true); +sql insert into tba4 values ('2022-09-29 15:15:03', 5, "5", false); +sql insert into tba4 values ('2022-09-29 15:15:04', null, null, null); +sql insert into tba5 values ('2022-09-30 15:15:01', 0, "a", false); +sql insert into tba5 values ('2022-09-30 15:15:02', 1, "0", true); +sql insert into tba5 values ('2022-09-30 15:15:03', 5, "5", false); +sql insert into tba5 values ('2022-09-30 15:15:04', null, null, null); + +print ======== case when xx + +sql select case when 3 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 4 then + return -1 +endi + +sql select case when 0 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when null then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 1 then 4+1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 5.000000000 then + return -1 +endi +if $data10 != 5.000000000 then + return -1 +endi +if $data20 != 5.000000000 then + return -1 +endi +if $data30 != 5.000000000 then + return -1 +endi + +sql select case when 1-1 then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 1+1 then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 1 then 1-1+2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 2.000000000 then + return -1 +endi +if $data30 != 2.000000000 then + return -1 +endi + +sql select case when 1 > 0 then 1 < 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when 1 > 2 then 1 < 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when abs(3) then abs(-1) end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when abs(1+1) then abs(-1)+abs(3) end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4.000000000 then + return -1 +endi +if $data10 != 4.000000000 then + return -1 +endi +if $data20 != 4.000000000 then + return -1 +endi +if $data30 != 4.000000000 then + return -1 +endi + +sql select case when 0 then 1 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 3 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 3 then + return -1 +endi + +sql select case when 0 then 1 when 1 then 0 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 0 then 1 when 1 then 0 when 2 then 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 'a' then 'b' when null then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when '2' then 'b' when null then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != b then + return -1 +endi +if $data10 != b then + return -1 +endi +if $data20 != b then + return -1 +endi +if $data30 != b then + return -1 +endi + +sql select case when 0 then 'b' else null end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 0 then 'b' else 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 2 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case when sum(2) then sum(2)-sum(1) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 4.000000000 then + return -1 +endi + +sql select case when sum(2) then abs(-2) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 2 then + return -1 +endi + +sql select case when ts then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != @22-09-26 15:15:01.000@ then + return -1 +endi +if $data10 != @22-09-26 15:15:02.000@ then + return -1 +endi +if $data20 != @22-09-26 15:15:03.000@ then + return -1 +endi +if $data30 != @22-09-26 15:15:04.000@ then + return -1 +endi + +sql select case when f1 then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != @22-09-26 15:15:02.000@ then + return -1 +endi +if $data20 != @22-09-26 15:15:03.000@ then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when f1 then f1 when f1 + 1 then f1 + 1 else f1 is null end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when f1 then 3 when ts then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1664176501000 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 1664176504000 then + return -1 +endi + +sql select case when 3 then f1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when f1 then 3 when 1 then 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case when sum(f1) then sum(f1)-abs(-1) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 5.000000000 then + return -1 +endi + +sql select case when sum(f1) then sum(f1)-abs(f1) end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != 0.000000000 then + return -1 +endi +if $data30 != 0.000000000 then + return -1 +endi + +sql select case when f1 then sum(f1) when f1 is not null then 9 else 8 end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != 8 then + return -1 +endi +if $data10 != 9 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 5 then + return -1 +endi + +sql select f1 from tba1 where f1 > case when f1 then 0 else 3 end; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 5 then + return -1 +endi + +sql select f1 from tba1 where ts > case when ts then ts end; +if $rows != 0 then + return -1 +endi + +sql select sum(f1),count(f1) from tba1 partition by case when f1 then f1 when 1 then 1 end; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +if $data11 != 1 then + return -1 +endi + +sql select case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end caseWhen, sum(f1),count(f1) from tba1 group by case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end order by caseWhen; +if $rows != 3 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data12 != 1 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data22 != 0 then + return -1 +endi + +sql select f1 from tba1 order by case when f1 <= 0 then 3 when f1 = 1 then 4 when f1 >= 3 then 2 else 1 end desc; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select cast(case f1 when f1 then f1 + 1 else f1 is null end as double) from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 6.000000000 then + return -1 +endi +if $data30 != 1.000000000 then + return -1 +endi + +sql select sum(case f1 when f1 then f1 + 1 else f1 is null end + 1) from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 14.000000000 then + return -1 +endi + +sql select case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end,sum(f1),count(f1) from tba1 state_window(case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end); +if $rows != 3 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data12 != 1 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data22 != 0 then + return -1 +endi + +sql select f1 from tba1 where case when case when f1 <= 0 then 3 when f1 = 1 then 4 when f1 >= 3 then 2 else 1 end > 2 then 1 else 0 end > 0; +if $rows != 2 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +sql select case when f1 is not null then case when f1 <= 0 then f1 else f1 * 10 end else -1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 10 then + return -1 +endi +if $data20 != 50 then + return -1 +endi +if $data30 != -1 then + return -1 +endi + + +print ======== case xx when xx + +sql select case 3 when 3 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 4 then + return -1 +endi + +sql select case 3 when 1 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case 3 when 1 then 4 else 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 2 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case 3 when null then 4 when '3' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case '3' when null then 4 when 3 then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case null when null then 4 when 3 then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case 3.0 when null then 4 when '3' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case f2 when 'a' then 4 when '0' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case f2 when f1 then f1 when f1 - 1 then f1 else 99 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 99 then + return -1 +endi + +sql select case cast(f2 as int) when 0 then f2 when f1 then 11 else ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != a then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 11 then + return -1 +endi +if $data30 != 1664176504 then + return -1 +endi + +sql select case f1 + 1 when 1 then 1 when 2 then 2 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 3 then + return -1 +endi + +sql select case f1 when sum(f1) then sum(f1)-abs(f1) end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != 0.000000000 then + return -1 +endi +if $data20 != 0.000000000 then + return -1 +endi +if $data30 != 0.000000000 then + return -1 +endi + +sql select f1, case sum(f1) when 1 then f1 + 99 when f1 then f1 -99 else f1 end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data11 != -99.000000000 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data21 != 100.000000000 then + return -1 +endi +if $data30 != 5 then + return -1 +endi +if $data31 != -94.000000000 then + return -1 +endi + +sql select case when 3 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when null then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1 then 4+1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1-1 then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1+1 then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when abs(3) then abs(-1) end from sta; +if $rows != 20 then + return -1 +endi +sql select case when abs(1+1) then abs(-1)+abs(3) end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 else 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 when 1 then 0 else 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 when 1 then 0 when 2 then 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 'a' then 'b' when null then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when '2' then 'b' when null then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 'b' else null end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 'b' else 2+abs(-2) end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 3 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 1 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 1 then 4 else 2 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when null then 4 when '3' then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case null when null then 4 when 3 then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3.0 when null then 4 when '3' then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select f2,case f2 when 'a' then 4 when '0' then 1 end from sta order by f2; +if $rows != 20 then + return -1 +endi +sql select f2,f1,case f2 when f1 then f1 when f1 - 1 then f1 else 99 end from sta order by f2; +if $rows != 20 then + return -1 +endi +sql select case cast(f2 as int) when 0 then f2 when f1 then 11 else ts end from sta; +if $rows != 20 then + return -1 +endi +sql select f1, case f1 + 1 when 1 then 1 when 2 then 2 else 3 end from sta order by f1; +if $rows != 20 then + return -1 +endi +sql select case f1 when sum(f1) then sum(f1)-abs(f1) end from sta group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != 0.000000000 then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi +sql select case sum(f1) when 1 then f1 + 99 when f1 then f1 -99 else f1 end from sta group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != -99.000000000 then + return -1 +endi +if $data20 != 1.000000000 then + return -1 +endi +if $data30 != 5.000000000 then + return -1 +endi + +sql select distinct tbname, case t1 when t2 then t1 else t1 + 100 end from sta order by tbname; +if $rows != 5 then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data31 != 101 then + return -1 +endi +if $data41 != 103 then + return -1 +endi + +sql_error select case when sum(f1) then sum(f1)-abs(f1) end from tba1; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 6fa7e39335..252ef10b1a 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -62,7 +62,16 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data -sql select * from c1 + +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select * from c1 -x steps + sql select * from stb sql select * from stb_1 sql select ts, c1, c2, c3 from c1 diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 11a387ed4d..89b6cadef8 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -158,9 +158,15 @@ print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v -sleep 1000 +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select avg(tbcol) as c from stb -x steps -sql select avg(tbcol) as c from stb sql select avg(tbcol) as c from stb where ts <= 1601481840000 sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 sql select avg(tbcol) as c from stb interval(1m) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1c6aef6286..cc85b8ddc8 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -# unstable python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3 @@ -248,8 +248,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 @@ -302,7 +302,7 @@ python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqCheckData1.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py +#python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqAlterSchema.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 8aafd747af..46eaf6cff1 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -325,24 +325,23 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { - printf(" **************************** How To Use TAB Key ********************************\n"); - printf(" * TDengine Command Line supports pressing TAB key to complete word, *\n"); - printf(" * including database name, table name, function name and keywords. *\n"); - printf(" * Press TAB key anywhere, you'll get surprise. *\n"); - printf(" * KEYBOARD SHORTCUT: *\n"); - printf(" * [ TAB ] ...... Complete the word or show help if no input *\n"); - printf(" * [ Ctrl + A ] ...... move cursor to [A]head of line *\n"); - printf(" * [ Ctrl + E ] ...... move cursor to [E]nd of line *\n"); - printf(" * [ Ctrl + W ] ...... move cursor to line of middle *\n"); - printf(" * [ Ctrl + L ] ...... clean screen *\n"); - printf(" * [ Ctrl + K ] ...... clean after cursor *\n"); - printf(" * [ Ctrl + U ] ...... clean before cursor *\n"); - printf(" * *\n"); + printf(" ****************************** Tab Completion **********************************\n"); + printf(" * The TDengine CLI supports tab completion for a variety of items, *\n"); + printf(" * including database names, table names, function names and keywords. *\n"); + printf(" * The full list of shortcut keys is as follows: *\n"); + printf(" * [ TAB ] ...... complete the current word *\n"); + printf(" * ...... if used on a blank line, display all valid commands *\n"); + printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n"); + printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n"); + printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n"); + printf(" * [ Ctrl + L ] ...... clear the entire screen *\n"); + printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n"); + printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n"); printf(" **********************************************************************************\n\n"); } void showHelp() { - printf("\nThe following are supported commands for TDengine Command Line:"); + printf("\nThe TDengine CLI supports the following commands:"); printf( "\n\ ----- A ----- \n\ diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index fd0ea60323..28375d8a35 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -998,7 +998,9 @@ void *shellCancelHandler(void *arg) { shell.stop_query = true; } else { #endif - taos_kill_query(shell.conn); + if (shell.conn) { + taos_kill_query(shell.conn); + } #ifdef WEBSOCKET } #endif