diff --git a/README-CN.md b/README-CN.md index 627b35698c..437d671bb9 100644 --- a/README-CN.md +++ b/README-CN.md @@ -107,7 +107,7 @@ sudo yum config-manager --set-enabled Powertools ### macOS ``` -sudo brew install argp-standalone pkgconfig +brew install argp-standalone pkgconfig ``` ### 设置 golang 开发环境 @@ -276,7 +276,7 @@ sudo make install 安装成功后，可以在应用程序中双击 TDengine 图标启动服务，或者在终端中启动 TDengine 服务： ```bash -launchctl start taosd +launchctl start com.tdengine.taosd ``` 用户可以使用 TDengine CLI 来连接 TDengine 服务，在终端中，输入： diff --git a/README.md b/README.md index 00d36efb5a..8d2567a816 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ sudo yum config-manager --set-enabled powertools ### macOS ``` -sudo brew install argp-standalone pkgconfig +brew install argp-standalone pkgconfig ``` ### Setup golang environment @@ -280,7 +280,7 @@ Installing from source code will also configure service management for TDengine. To start the service after installation, double-click the /applications/TDengine to start the program, or in a terminal, use: ```bash -launchctl start taosd +launchctl start com.tdengine.taosd ``` Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: diff --git a/cmake/cmake.platform b/cmake/cmake.platform index e4d440d76e..3e239d2e0c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -45,10 +45,19 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.") - IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") - MESSAGE("Current system arch is 64") + IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") + MESSAGE("Current system arch is arm64") SET(TD_DARWIN_64 TRUE) + SET(TD_DARWIN_ARM64 TRUE) ADD_DEFINITIONS("-D_TD_DARWIN_64") + ADD_DEFINITIONS("-D_TD_DARWIN_ARM64") + ENDIF () + IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") + MESSAGE("Current system arch is x86_64") + SET(TD_DARWIN_64 TRUE) + SET(TD_DARWIN_X64 TRUE) + ADD_DEFINITIONS("-D_TD_DARWIN_64") + ADD_DEFINITIONS("-D_TD_DARWIN_X64") ENDIF () ADD_DEFINITIONS("-DHAVE_UNISTD_H") diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 7bde332c8c..73409a950a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d58230c + GIT_TAG 4d02980 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a1a6cd4519..969724cc8d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -270,7 +270,7 @@ if(${JEMALLOC_ENABLED}) PREFIX "jemalloc" SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls --with-malloc-conf='background_thread:true,metadata_thp:auto' BUILD_COMMAND ${MAKE} ) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index f734e354fb..7800fb1c88 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -189,13 +189,13 @@ After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengin -After the installation is complete, double-click the /applications/TDengine to start the program, or run `launchctl start taosd` to start TDengine Server. +After the installation is complete, double-click the /applications/TDengine to start the program, or run `launchctl start com.tdengine.taosd` to start TDengine Server. The following `launchctl` commands can help you manage TDengine service: -- Start TDengine Server: `launchctl start taosd` +- Start TDengine Server: `launchctl start com.tdengine.taosd` -- Stop TDengine Server: `launchctl stop taosd` +- Stop TDengine Server: `launchctl stop com.tdengine.taosd` - Check TDengine Server status: `launchctl list | grep taosd` diff --git a/docs/en/12-taos-sql/28-recovery.md b/docs/en/12-taos-sql/28-recovery.md index 14ac14f867..e869ffc45f 100644 --- a/docs/en/12-taos-sql/28-recovery.md +++ b/docs/en/12-taos-sql/28-recovery.md @@ -16,10 +16,10 @@ You can use the SHOW CONNECTIONS statement to find the conn_id. ## Terminate a Query ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -You can use the SHOW QUERIES statement to find the query_id. +You can use the SHOW QUERIES statement to find the kill_id. ## Terminate a Transaction diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index d95086d4c4..6295d8553d 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js - **name**: specify the name of the database. -- **drop**: indicate whether to delete the database before inserting. The default is true. +- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop. #### Stream processing related configuration parameters diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index d91675a0ab..8053c4d25d 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -675,7 +675,7 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Meaning | Whether to generate core file when server crashes | | Value Range | 0: false, 1: true | | Default Value | 1 | -| Note | The core file is generated under root directory `systemctl/launchctl start taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | +| Note | The core file is generated under root directory `systemctl start taosd`/`launchctl start com.tdengine.taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | ### udf diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index 6622ee525a..a938084eb1 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -181,7 +181,7 @@ To make full use of the characteristics of time-series data, TDengine splits the For time-series data, there is generally a retention policy, which is determined by the system configuration parameter `keep`. Data files exceeding this set number of days will be automatically deleted by the system to free up storage space. -Given `duration` and `keep` parameters, the total number of data files in a vnode is: keep/duration. The total number of data files should not be too large or too small. 10 to 100 is appropriate. Based on this principle, reasonable `duration` can be set. In the current version, parameter `keep` can be modified, but parameter `duration` cannot be modified once it is set. +Given `duration` and `keep` parameters, the total number of data files in a vnode is: round up of (keep/duration+1). The total number of data files should not be too large or too small. 10 to 100 is appropriate. Based on this principle, reasonable `duration` can be set. In the current version, parameter `keep` can be modified, but parameter `duration` cannot be modified once it is set. In each data file, the data of a table is stored in blocks. A table can have one or more data file blocks. In a file block, data is stored in columns, occupying a continuous storage space, thus greatly improving the reading speed. The size of file block is determined by the system parameter `maxRows` (the maximum number of records per block), and the default value is 4096. This value should not be too large or too small. If it is too large, data location for queries will take a longer time. If it is too small, the index of data block is too large, and the compression efficiency will be low with slower reading speed. diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index cee036454e..836beb2417 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -1,6 +1,58 @@ import taos -from taos.tmq import TaosConsumer -consumer = TaosConsumer('topic_ctb_column', group_id='vg2') -for msg in consumer: - for row in msg: - print(row) +from taos.tmq import * + +conn = taos.connect() + +print("init") +conn.execute("drop database if exists py_tmq") +conn.execute("create database if not exists py_tmq vgroups 2") +conn.select_db("py_tmq") +conn.execute( + "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)" +) +conn.execute("create table if not exists tb1 using stb1 tags(1)") +conn.execute("create table if not exists tb2 using stb1 tags(2)") +conn.execute("create table if not exists tb3 using stb1 tags(3)") + +print("create topic") +conn.execute("drop topic if exists topic_ctb_column") +conn.execute( + "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" +) + +print("build consumer") +conf = TaosTmqConf() +conf.set("group.id", "tg2") +conf.set("td.connect.user", "root") +conf.set("td.connect.pass", "taosdata") +conf.set("enable.auto.commit", "true") + + +def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + + +conf.set_auto_commit_cb(tmq_commit_cb_print, None) +tmq = conf.new_consumer() + +print("build topic list") + +topic_list = TaosTmqList() +topic_list.append("topic_ctb_column") + +print("basic consume loop") +tmq.subscribe(topic_list) + +sub_list = tmq.subscription() + +print("subscribed topics: ", sub_list) + +while 1: + res = tmq.poll(1000) + if res: + topic = res.get_topic_name() + vg = res.get_vgroup_id() + db = res.get_db_name() + print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") + for row in res: + print(row) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 50ee721386..167a68d3c8 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -188,13 +188,13 @@ Active: inactive (dead) -安装后，在应用程序目录下，双击 TDengine 图标来启动程序，也可以运行 `launchctl start taosd` 来启动 TDengine 服务进程。 +安装后，在应用程序目录下，双击 TDengine 图标来启动程序，也可以运行 `launchctl start com.tdengine.taosd` 来启动 TDengine 服务进程。 如下 `launchctl` 命令可以帮助你管理 TDengine 服务： -- 启动服务进程：`launchctl start taosd` +- 启动服务进程：`launchctl start com.tdengine.taosd` -- 停止服务进程：`launchctl stop taosd` +- 停止服务进程：`launchctl stop com.tdengine.taosd` - 查看服务状态：`launchctl list | grep taosd` diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 4c33b5232a..9346c3f763 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -340,7 +340,7 @@ LTRIM(expr) #### RTRIM ```sql -LTRIM(expr) +RTRIM(expr) ``` **功能说明**：返回清除右边空格后的字符串。 diff --git a/docs/zh/12-taos-sql/28-recovery.md b/docs/zh/12-taos-sql/28-recovery.md index 582c373907..b5088e7982 100644 --- a/docs/zh/12-taos-sql/28-recovery.md +++ b/docs/zh/12-taos-sql/28-recovery.md @@ -17,10 +17,10 @@ conn_id 可以通过 `SHOW CONNECTIONS` 获取。 ## 终止查询 ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -query_id 可以通过 `SHOW QUERIES` 获取。 +kill_id 可以通过 `SHOW QUERIES` 获取。 ## 终止事务 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 7da8e9f331..1ab7f4f016 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 + + +## SCHEMALESS 变更 + +| # | **元素** | **
差异性
** | **说明** | +| - | :------- | :-------- | :------- | +| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。 diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 9f4f728f78..e067c646b2 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **name** : 数据库名。 -- **drop** : 插入前是否删除数据库，默认为 true。 +- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。 #### 流式计算相关配置参数 diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 07d5b0692b..b89f625376 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -115,7 +115,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 ## 数据模式映射规则 -本节将说明行协议的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 +本节将说明 InfluxDB 行协议(Line Protocol)的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 超级表名称。tag_set 中的 标签名称为 数据模式中的标签名，field_set 中的名称为列名称。以如下数据为例，说明映射规则： ```json diff --git a/examples/c/demoapi.c b/examples/c/demoapi.c index c3a6d26776..18f1b5a059 100644 --- a/examples/c/demoapi.c +++ b/examples/c/demoapi.c @@ -1,5 +1,17 @@ -// C api call sequence demo -// to compile: gcc -o apidemo apidemo.c -ltaos +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include #include @@ -13,23 +25,23 @@ #include "taos.h" #define debugPrint(fmt, ...) \ - do { if (g_args.debug_print || g_args.verbose_print) \ - fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0) + do { if (g_args.debug_print || g_args.verbose_print) {\ + fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0) #define warnPrint(fmt, ...) \ do { fprintf(stderr, "\033[33m"); \ fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define errorPrint(fmt, ...) \ do { fprintf(stderr, "\033[31m"); \ fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define okPrint(fmt, ...) \ do { fprintf(stderr, "\033[32m"); \ fprintf(stderr, "OK: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) int64_t g_num_of_tb = 2; int64_t g_num_of_rec = 3; @@ -74,10 +86,16 @@ static void prepare_data(TAOS* taos) { res = taos_query(taos, "create database test;"); taos_free_result(res); taosMsleep(100); - taos_select_db(taos, "test"); + if (taos_select_db(taos, "test")) { + errorPrint("%s() LN%d: taos_select_db() failed\n", + __func__, __LINE__); + return; + } char command[1024] = {0}; - sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));"); + sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, " + "bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, " + "city binary(20), dist nchar(20), street binary(20));"); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { okPrint("%s created\n", "meters"); @@ -117,12 +135,11 @@ static void prepare_data(TAOS* taos) { sprintf(command, "insert into t%"PRId64" " "values(%" PRId64 ", %f, %"PRId64", " "'%c%d', '%s%c%d', '%c%d')", - i, 1650000000000+j, (float)j, j, + i, 1650000000000+j, j * 1.0, j, 'a'+(int)j%25, rand(), // "涛思", 'z' - (int)j%25, rand(), "TAOS", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -150,8 +167,7 @@ static void prepare_data(TAOS* taos) { i, 1650000000000+j+1, (float)j, j, 'a'+(int)j%25, rand(), "数据", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -179,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { } if (block) { - warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n"); + warnPrint("%s", "call taos_fetch_block(), " + "don't call taos_fetch_lengths()\n"); int rows = 0; while ((rows = taos_fetch_block(res, &row))) { int *lengths = taos_fetch_lengths(res); @@ -190,7 +207,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d type is %d, no need get offset\n", f, fields[f].type); for (int64_t c = 0; c < rows; c++) { - switch(fields[f].type) { + switch (fields[f].type) { case TSDB_DATA_TYPE_TIMESTAMP: if (taos_is_null(res, c, f)) { printf("col%d, row: %"PRId64" " @@ -199,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %"PRId64"\n", f, c, - *(int64_t*)((char*)(row[f])+c*sizeof(int64_t))); + *(int64_t*)((char*)(row[f]) + +c*sizeof(int64_t))); } break; @@ -211,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %d\n", f, c, - *(int32_t*)((char*)(row[f])+c*sizeof(int32_t))); + *(int32_t*)((char*)(row[f]) + +c*sizeof(int32_t))); } break; @@ -223,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %f\n", f, c, - *(float*)((char*)(row[f])+c*sizeof(float))); + *(float*)((char*)(row[f]) + +c*sizeof(float))); } break; @@ -238,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { if (offsets) { for (int c = 0; c < rows; c++) { if (offsets[c] != -1) { - int length = *(int16_t*)((char*)(row[f]) + offsets[c]); + int length = *(int16_t*)((char*)(row[f]) + + offsets[c]); char *buf = calloc(1, length + 1); - strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length); - printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n", + strncpy(buf, (char *)((char*)(row[f]) + + offsets[c] + 2), length); + printf("row: %d, col: %d, offset: %d, " + "length: %d, content: %s\n", c, f, offsets[c], length, buf); free(buf); } else { - printf("row: %d, col: %d, offset: -1, means content is NULL\n", + printf("row: %d, col: %d, offset: -1, " + "means content is NULL\n", c, f); } } @@ -267,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { int* lengths = taos_fetch_lengths(res); if (lengths) { for (int c = 0; c < num_fields; c++) { - printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n", + printf("row: %"PRId64", col: %d, is_null: %s, " + "length of column %d is %d\n", num_rows, c, taos_is_null(res, num_rows, c)?"True":"False", c, lengths[c]); @@ -277,7 +302,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { __func__, __LINE__, tbname); } - num_rows ++; + num_rows++; } } @@ -322,7 +347,8 @@ int main(int argc, char *argv[]) { #endif TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { - printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); + printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", + taos_errstr(taos)); exit(1); } diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 0e22985059..fd0521af3b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -93,6 +93,7 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in // query client extern int32_t tsQueryPolicy; +extern int32_t tsQueryRspPolicy; extern int32_t tsQuerySmaOptimize; extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9f21ee007f..748bd6afcc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { if (pRebInfo == NULL) { return NULL; } - strcpy(pRebInfo->key, key); + tstrncpy(pRebInfo->key, key, sizeof(pRebInfo->key)); pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->lostConsumers == NULL) { goto _err; diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 0d43539629..cdb1642a5c 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -30,9 +30,10 @@ typedef struct SMnode SMnode; typedef struct { int32_t dnodeId; - bool standby; bool deploy; - SReplica replica; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; SMsgCb msgCb; } SMnodeOpt; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c179d69960..7d95e840e9 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -54,6 +54,9 @@ typedef enum { #define QUERY_POLICY_QNODE 3 #define QUERY_POLICY_CLIENT 4 +#define QUERY_RSP_POLICY_DELAY 0 +#define QUERY_RSP_POLICY_QUICK 1 + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision @@ -230,7 +233,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, - void* (*mallocFp)(int32_t)); + void* (*mallocFp)(int64_t)); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 6322c6a1e9..ef8e80b57f 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -43,7 +43,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); -int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow); +int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows); /* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 411a63a379..0f8220f19c 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -211,7 +211,7 @@ typedef struct SSyncInfo { int32_t syncInit(); void syncCleanUp(); -int64_t syncOpen(const SSyncInfo* pSyncInfo); +int64_t syncOpen(SSyncInfo* pSyncInfo); void syncStart(int64_t rid); void syncStop(int64_t rid); int32_t syncSetStandby(int64_t rid); @@ -233,7 +233,7 @@ const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot); -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg); // build SRpcMsg, need to call syncPropose with SRpcMsg int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 56c5750475..53edcd4fee 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -123,9 +123,9 @@ void rpcCleanup(); void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void rpcCloseImpl(void *); -void *rpcMallocCont(int32_t contLen); +void *rpcMallocCont(int64_t contLen); void rpcFreeCont(void *pCont); -void *rpcReallocCont(void *ptr, int32_t contLen); +void *rpcReallocCont(void *ptr, int64_t contLen); // Because taosd supports multi-process mode // These functions should not be used on the server side diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index de31a970df..adf244e32a 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -43,6 +43,7 @@ extern "C" { #define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL #define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) +#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE) typedef enum { TAOS_WAL_WRITE = 1, diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 293d9d17f8..d4e94d6173 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace; void osDefaultInit(); void osUpdate(); void osCleanup(); + bool osLogSpaceAvailable(); bool osDataSpaceAvailable(); bool osTempSpaceAvailable(); + +bool osLogSpaceSufficient(); +bool osDataSpaceSufficient(); +bool osTempSpaceSufficient(); + void osSetTimezone(const char *timezone); void osSetSystemLocale(const char *inLocale, const char *inCharSet); @@ -59,4 +65,4 @@ void osSetSystemLocale(const char *inLocale, const char *inCharSet); } #endif -#endif /*_TD_OS_ENV_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENV_H_*/ diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 3146590d6c..7ec8366ee4 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -29,12 +29,12 @@ extern "C" { #define free FREE_FUNC_TAOS_FORBID #endif -void *taosMemoryMalloc(int32_t size); -void *taosMemoryCalloc(int32_t num, int32_t size); -void *taosMemoryRealloc(void *ptr, int32_t size); +void *taosMemoryMalloc(int64_t size); +void *taosMemoryCalloc(int64_t num, int64_t size); +void *taosMemoryRealloc(void *ptr, int64_t size); void *taosMemoryStrDup(const char *ptr); void taosMemoryFree(void *ptr); -int32_t taosMemorySize(void *ptr); +int64_t taosMemorySize(void *ptr); void taosPrintBackTrace(); #define taosMemoryFreeClear(ptr) \ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6bc0e0e7dd..bbddb539c6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -450,6 +450,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) +#define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) // tfs #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index f6d37e3d75..7c93c8dd8d 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -26,7 +26,7 @@ typedef struct SRBTree SRBTree; typedef struct SRBTreeNode SRBTreeNode; typedef struct SRBTreeIter SRBTreeIter; -typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); +typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *); // SRBTree ============================================= #define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min) @@ -36,7 +36,7 @@ void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn); SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z); void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode); // SRBTreeIter ============================================= #define tRBTreeIterCreate(tree, ascend) \ @@ -53,8 +53,6 @@ struct SRBTreeNode { SRBTreeNode *right; }; -#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) - struct SRBTree { tRBTreeCmprFn cmprFn; int64_t n; diff --git a/packaging/release.sh b/packaging/release.sh index 9ebbc50a84..ce42491892 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -260,7 +260,7 @@ if [ "$osType" != "Darwin" ]; then if [[ "$pagMode" == "full" ]]; then if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') + taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" ${csudo}./make-taos-tools-deb.sh ${top_dir} \ @@ -285,7 +285,7 @@ if [ "$osType" != "Darwin" ]; then if [[ "$pagMode" == "full" ]]; then if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then cd ${top_dir}/tools/taos-tools/packaging/rpm - taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g') + taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" ${csudo}./make-taos-tools-rpm.sh ${top_dir} \ diff --git a/packaging/tools/TDengine b/packaging/tools/TDengine index 3025a5639f..c0b80564c4 100755 --- a/packaging/tools/TDengine +++ b/packaging/tools/TDengine @@ -17,12 +17,12 @@ EOF taosd_status=`Launchctl list | grep taosd | head -n 1 | awk '{print $1}'` if [ "$taosd_status"x = "-"x ]; then - launchctl start taosd + launchctl start com.tdengine.taosd showAlertMessage "Taosd is running!" "TDengine" "ok" "note" else choose_result=`showAlertMessage "Taosd is running!\nDo you want to close it?" "TDengine" "yes,cancel" "stop"` if [ "$choose_result"x = "button returned:yes"x ]; then - launchctl stop taosd + launchctl stop com.tdengine.taosd fi fi diff --git a/packaging/tools/TDengine.pkgproj b/packaging/tools/TDengine.pkgproj new file mode 100644 index 0000000000..7aaac62735 --- /dev/null +++ b/packaging/tools/TDengine.pkgproj @@ -0,0 +1,882 @@ + + + + + PACKAGES + + + MUST-CLOSE-APPLICATION-ITEMS + + MUST-CLOSE-APPLICATIONS + + PACKAGE_FILES + + DEFAULT_INSTALL_LOCATION + / + HIERARCHY + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Applications + PATH_TYPE + 0 + PERMISSIONS + 509 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Application Support + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Automator + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Documentation + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Extensions + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Filesystems + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Frameworks + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Input Methods + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Internet Plug-Ins + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Keyboard Layouts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchAgents + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchDaemons + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PreferencePanes + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Preferences + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 80 + PATH + Printers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PrivilegedHelperTools + PATH_TYPE + 0 + PERMISSIONS + 1005 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickLook + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickTime + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Screen Savers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Scripts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Services + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Widgets + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + Library + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 0 + PATH + Shared + PATH_TYPE + 0 + PERMISSIONS + 1023 + TYPE + 1 + UID + 0 + + + GID + 80 + PATH + Users + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + / + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + PAYLOAD_TYPE + 0 + PRESERVE_EXTENDED_ATTRIBUTES + + SHOW_INVISIBLE + + SPLIT_FORKS + + TREAT_MISSING_FILES_AS_WARNING + + VERSION + 5 + + PACKAGE_SCRIPTS + + POSTINSTALL_PATH + + PATH + /opt/taos/TDengine/packaging/tools/post.sh + PATH_TYPE + 0 + + PREINSTALL_PATH + + PATH_TYPE + 0 + + RESOURCES + + + CHILDREN + + GID + 0 + PATH + /opt/tdengine + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 3 + UID + 0 + + + + PACKAGE_SETTINGS + + AUTHENTICATION + 1 + CONCLUSION_ACTION + 0 + FOLLOW_SYMBOLIC_LINKS + + IDENTIFIER + com.taosdata.pkg.pkgtaos + LOCATION + 0 + NAME + TDengine + OVERWRITE_PERMISSIONS + + PAYLOAD_SIZE + -1 + REFERENCE_PATH + + RELOCATABLE + + USE_HFS+_COMPRESSION + + VERSION + 3.0.1.4 + + TYPE + 0 + UUID + 48F6B249-AF56-46E3-B75A-AEED5858A764 + + + PROJECT + + PROJECT_COMMENTS + + NOTES + + + + PROJECT_PRESENTATION + + BACKGROUND + + APPAREANCES + + DARK_AQUA + + LIGHT_AQUA + + + SHARED_SETTINGS_FOR_ALL_APPAREANCES + + + INSTALLATION TYPE + + HIERARCHIES + + INSTALLER + + LIST + + + CHILDREN + + DESCRIPTION + + OPTIONS + + HIDDEN + + STATE + 1 + + PACKAGE_UUID + 48F6B249-AF56-46E3-B75A-AEED5858A764 + TITLE + + TYPE + 0 + UUID + 7ED88D2C-D55C-46FF-99CB-73313ACAD73D + + + REMOVED + + + + MODE + 0 + + INSTALLATION_STEPS + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewIntroductionController + INSTALLER_PLUGIN + Introduction + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewReadMeController + INSTALLER_PLUGIN + ReadMe + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewLicenseController + INSTALLER_PLUGIN + License + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewDestinationSelectController + INSTALLER_PLUGIN + TargetSelect + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewInstallationTypeController + INSTALLER_PLUGIN + PackageSelection + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewInstallationController + INSTALLER_PLUGIN + Install + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewSummaryController + INSTALLER_PLUGIN + Summary + LIST_TITLE_KEY + InstallerSectionTitle + + + INTRODUCTION + + LOCALIZATIONS + + + LANGUAGE + English + VALUE + + PATH + /opt/taos/TDengine/packaging/tools/mac_before_install.txt + PATH_TYPE + 0 + + + + + LICENSE + + LOCALIZATIONS + + MODE + 0 + + README + + LOCALIZATIONS + + + SUMMARY + + LOCALIZATIONS + + + TITLE + + LOCALIZATIONS + + + LANGUAGE + English + VALUE + TDengine + + + + + PROJECT_REQUIREMENTS + + LIST + + RESOURCES + + ROOT_VOLUME_ONLY + + + PROJECT_SETTINGS + + BUILD_FORMAT + 0 + BUILD_PATH + + PATH + /opt/taos/TDengine/release + PATH_TYPE + 0 + + EXCLUDED_FILES + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .DS_Store + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .DS_Store files + PROXY_TOOLTIP + Remove ".DS_Store" files created by the Finder. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .pbdevelopment + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .pbdevelopment files + PROXY_TOOLTIP + Remove ".pbdevelopment" files created by ProjectBuilder or Xcode. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + CVS + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .cvsignore + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .cvspass + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .svn + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .git + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .gitignore + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove SCM metadata + PROXY_TOOLTIP + Remove helper files and folders used by the CVS, SVN or Git Source Code Management systems. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + classes.nib + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + designable.db + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + info.nib + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Optimize nib files + PROXY_TOOLTIP + Remove "classes.nib", "info.nib" and "designable.nib" files within .nib bundles. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + Resources Disabled + TYPE + 1 + + + PROTECTED + + PROXY_NAME + Remove Resources Disabled folders + PROXY_TOOLTIP + Remove "Resources Disabled" folders. + STATE + + + + SEPARATOR + + + + NAME + TDengine-client-3.0.1.4-macOS-arm64 + PAYLOAD_ONLY + + TREAT_MISSING_PRESENTATION_DOCUMENTS_AS_WARNING + + + + TYPE + 0 + VERSION + 2 + + diff --git a/packaging/tools/com.taosdata.taosd.plist b/packaging/tools/com.taosdata.taosd.plist index ca9b8c5ef1..0bf3135ac3 100644 --- a/packaging/tools/com.taosdata.taosd.plist +++ b/packaging/tools/com.taosdata.taosd.plist @@ -3,7 +3,7 @@ Label - taosd + com.tdengine.taosd ProgramArguments /usr/local/bin/taosd diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index f70ea7161f..b33cd478b8 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,5 +1,5 @@ TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. To configure TDengine : edit /etc/taos/taos.cfg -To start service : launchctl start taosd +To start service : launchctl start com.tdengine.taosd To access TDengine : use taos in shell \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index b8796ddf09..b56a5a203d 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -615,7 +615,7 @@ function update_TDengine() { if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" @@ -666,7 +666,7 @@ function install_TDengine() { if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 29d4ac017c..b729513b97 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -18,6 +18,7 @@ if [ "$osType" != "Darwin" ]; then script_dir=$(dirname $(readlink -f "$0")) verNumber="" lib_file_ext="so" + lib_file_ext_1="so.1" bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" @@ -29,6 +30,7 @@ else script_dir=${source_dir}/packaging/tools verNumber=`ls tdengine/driver | grep -E "libtaos\.[0-9]\.[0-9]" | sed "s/libtaos.//g" | sed "s/.dylib//g" | head -n 1` lib_file_ext="dylib" + lib_file_ext_1="1.dylib" bin_link_dir="/usr/local/bin" lib_link_dir="/usr/local/lib" @@ -134,14 +136,14 @@ function install_lib() { [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} + ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib_link_dir}/libtaosws.${lib_file_ext} ||: - if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} || : [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} || : fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6172fc9bc8..bace0bc869 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -117,7 +117,7 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : + [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index e8e3c87849..0c445c7fbf 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -19,6 +19,13 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) +if(TD_DARWIN_ARM64) + target_link_libraries( + taos + PRIVATE "-arch x86_64" + ) +endif() + if(TD_WINDOWS) INCLUDE_DIRECTORIES(jni/windows) INCLUDE_DIRECTORIES(jni/windows/win32) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 096e2cdac8..3f24a08cf4 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + tscTrace("conn %u hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7b437e4769..b494140da5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -186,7 +186,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, STscObj* pTscObj = (*pRequest)->pTscObj; if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { - tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + tscError("%" PRIx64 " failed to add to request container, reqId:0x%" PRIu64 ", conn:%" PRIx64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); taosMemoryFree(param); @@ -371,7 +371,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) { pInfo->pQnodeList = taosArrayDup(pNodeList); taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad); tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId, - taosArrayGetSize(pInfo->pQnodeList)); + (int)taosArrayGetSize(pInfo->pQnodeList)); } taosThreadMutexUnlock(&pInfo->qnodeMutex); @@ -1036,30 +1036,38 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->type = pQuery->msgType; SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); + SPlanContext cxt = {.queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite, - .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, - .pUser = pRequest->pTscObj->user, - .sysInfo = pRequest->pTscObj->sysInfo, - .allocatorId = pRequest->allocatorRefId}; - SQueryPlan* pDag = NULL; - int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); - if (code) { - tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); - } else { - pRequest->body.subplanNum = pDag->numOfSubplans; + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user, + .sysInfo = pRequest->pTscObj->sysInfo, + .allocatorId = pRequest->allocatorRefId}; + + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SQueryPlan* pDag = NULL; + + int64_t st = taosGetTimestampUs(); + int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); + if (code) { + tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + pRequest->body.subplanNum = pDag->numOfSubplans; + } + + pRequest->metric.planEnd = taosGetTimestampUs(); + if (code == TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self, + (pRequest->metric.planEnd - st)/1000.0, pRequest->requestId); } - - pRequest->metric.planEnd = taosGetTimestampUs(); - - if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { - SArray* pNodeList = NULL; - buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { + SArray* pNodeList = NULL; + buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter, .requestId = pRequest->requestId, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6f8cef7c0d..6126817ece 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL - : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL) + return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) : (const char *)tstrerror(pRequest->code); } } @@ -222,7 +221,9 @@ void taos_kill_query(TAOS *taos) { int64_t rid = *(int64_t *)taos; STscObj *pTscObj = acquireTscObj(rid); - stopAllRequests(pTscObj->pRequests); + if (pTscObj) { + stopAllRequests(pTscObj->pRequests); + } releaseTscObj(rid); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a257335931..1dc319be94 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -410,6 +410,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVAlterTbReq vAlterTbReq = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -419,7 +420,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -524,6 +525,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropStbReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -533,7 +535,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -556,6 +558,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropTbBatchReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -565,7 +568,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -684,7 +687,7 @@ end: static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; - SDecoder coder; + SDecoder coder = {0}; SMDropStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; @@ -1212,6 +1215,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; + SSubmitReq* subReq = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); if (!pRequest) { @@ -1228,8 +1232,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - strcpy(pName.dbname, pRequest->pDb); - strcpy(pName.tname, tbname); + tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); + tstrncpy(pName.tname, tbname, sizeof(pName.tname)); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1278,7 +1282,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; int32_t totalLen = sizeof(SSubmitReq) + submitLen; - SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); + subReq = taosMemoryCalloc(1, totalLen); SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); @@ -1352,6 +1356,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) if (NULL == pQuery) { uError("create SQuery error"); code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(subReq); goto end; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; @@ -1390,6 +1395,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); + taosMemoryFree(subReq); return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c5633e73d0..5c37822222 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -299,6 +299,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); return -1; } } @@ -430,7 +431,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { STableMeta *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -874,7 +875,8 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); + return TSDB_CODE_SUCCESS; } @@ -1009,6 +1011,7 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t * static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; const char *sql = data; size_t childTableNameLen = strlen(tsSmlChildTableName); while (*sql != '\0') { @@ -1082,7 +1085,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab kv->length = valueLen; kv->type = TSDB_DATA_TYPE_NCHAR; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); } return TSDB_CODE_SUCCESS; @@ -1370,8 +1373,14 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { SHashObj *s2 = *(SHashObj **)key2; SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); + if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv1"); + return -1; + } + if(!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv2"); + return -1; + } if (kv1->i < kv2->i) { return -1; } else if (kv1->i > kv2->i) { @@ -1735,7 +1744,7 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { kv->i = tsVal; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } @@ -1932,6 +1941,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { } static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1941,7 +1951,7 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { if (!kv) { return TSDB_CODE_OUT_OF_MEMORY; } - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); kv->key = VALUE; kv->keyLen = VALUE_LEN; @@ -1955,7 +1965,9 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { int32_t ret = TSDB_CODE_SUCCESS; - + if (!pKVs){ + return TSDB_CODE_OUT_OF_MEMORY; + } cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1985,14 +1997,14 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN return TSDB_CODE_TSC_INVALID_JSON; } memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); + tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); continue; } // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (pKVs) taosArrayPush(pKVs, &kv); + taosArrayPush(pKVs, &kv); // key kv->keyLen = keyLen; @@ -2103,6 +2115,8 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { if (!oneTable) { tinfo = smlBuildTableInfo(); if (!tinfo) { + smlDestroyCols(cols); + if (info->dataFormat) taosArrayDestroy(cols); return TSDB_CODE_TSC_OUT_OF_MEMORY; } taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); @@ -2295,7 +2309,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlTableInfo *tableData = *oneTable; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); SRequestConnInfo conn = {0}; @@ -2477,11 +2491,12 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { } else { pParam->request->body.resInfo.numOfRows += info->affectedRows; } + // unlock + taosThreadSpinUnlock(&pParam->lock); + if (pParam->cnt == pParam->total) { tsem_post(&pParam->sem); } - taosThreadSpinUnlock(&pParam->lock); - // unlock uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); info->cost.endTime = taosGetTimestampUs(); info->cost.code = code; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 7a800bd334..bf3fd00f14 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -201,6 +201,9 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if(!pSrc){ + return TSDB_CODE_OUT_OF_MEMORY; + } STableDataBlocks* pDst = NULL; STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 66f992f05f..02aa7b806c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -709,6 +709,7 @@ void tmqSendHbReq(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { + taosMemoryFree(param); return; } int64_t consumerId = tmq->consumerId; @@ -870,8 +871,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("consumer setup failed since %s", terrstr()); return NULL; } @@ -939,10 +939,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { return NULL; } - int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); - *pRefId = pTmq->refId; - if (pTmq->hbBgEnable) { + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = pTmq->refId; pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } @@ -966,6 +965,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = -1; + tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); + req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); @@ -1297,7 +1298,8 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { pParam->code = code; if (code != 0) { - tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); + tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d, code %x", tmq->consumerId, + pParam->async, code); goto END; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69174476a..21a52a4b57 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) { int32_t n = 0; char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t* length = taos_fetch_lengths(pRes); - for (int32_t i = 0; i < numOfFields; ++i) { - printf("(%d):%d ", i, length[i]); - } - printf("\n"); - - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - memset(str, 0, sizeof(str)); +// int32_t* length = taos_fetch_lengths(pRes); +// for(int32_t i = 0; i < numOfFields; ++i) { +// printf("(%d):%d " , i, length[i]); +// } +// printf("\n"); +// +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// memset(str, 0, sizeof(str)); } } @@ -102,17 +102,6 @@ void queryCallback(void* param, void* res, int32_t code) { taos_fetch_raw_block_a(res, fetchCallback, param); } -void queryCallback1(void* param, void* res, int32_t code) { - if (code != TSDB_CODE_SUCCESS) { - printf("failed to execute, reason:%s\n", taos_errstr(res)); - } - - taos_free_result(res); - - printf("exec query:\n"); - taos_query_a(param, "select * from tm1", queryCallback, param); -} - void createNewTable(TAOS* pConn, int32_t index) { char str[1024] = {0}; sprintf(str, "create table tu%d using st2 tags(%d)", index, index); @@ -123,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for (int32_t i = 0; i < 3280; i += 20) { + for(int32_t i = 0; i < 10000; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -141,10 +130,49 @@ void createNewTable(TAOS* pConn, int32_t index) { taos_free_result(p); } } + +void *queryThread(void *arg) { + TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to db, reason:%s", taos_errstr(pConn)); + return NULL; + } + + int64_t el = 0; + + for (int32_t i = 0; i < 5000000; ++i) { + int64_t st = taosGetTimestampUs(); + TAOS_RES* pRes = taos_query(pConn, + "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;"); + if (taos_errno(pRes) != 0) { + printf("failed, reason:%s\n", taos_errstr(pRes)); + } else { + printResult(pRes); + } + + taos_free_result(pRes); + el += (taosGetTimestampUs() - st); + if (i % 1000 == 0 && i != 0) { + printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000)); + } + } + + taos_close(pConn); + return NULL; +} + +static int32_t numOfThreads = 1; } // namespace int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + numOfThreads = atoi(argv[1]); + } + + numOfThreads = TMAX(numOfThreads, 1); + printf("the runing threads is:%d", numOfThreads); + return RUN_ALL_TESTS(); } @@ -664,7 +692,6 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } -#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -676,13 +703,14 @@ TEST(testCase, projection_query_tables) { // } // taos_free_result(pRes); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu"); taos_free_result(pRes); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } + taos_free_result(pRes); pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); @@ -722,6 +750,16 @@ TEST(testCase, projection_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif + +TEST(testCase, tsbs_perf_test) { + TdThread qid[20] = {0}; + + for(int32_t i = 0; i < numOfThreads; ++i) { + taosThreadCreate(&qid[i], NULL, queryThread, NULL); + } + getchar(); +} #if 0 TEST(testCase, projection_query_stables) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad5772e0fe..269e72f151 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -916,6 +916,10 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { + if(!pTag || !pTagVal){ + return false; + } + int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; int16_t midx; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 88df54a9ea..56daa4075a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -90,6 +90,7 @@ bool tsSmlDataFormat = false; // query int32_t tsQueryPolicy = 1; +int32_t tsQueryRspPolicy = 0; int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -350,6 +351,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1; if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; @@ -728,6 +730,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; + tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index a4c37dd334..17cbde437b 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -41,8 +41,8 @@ typedef struct SMnodeMgmt { } SMnodeMgmt; // mmFile.c -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); +int32_t mmReadFile(const char *path, SMnodeOpt *pOption); +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption); // mmHandle.c SArray *mmGetMsgHandles(); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 27a35ae17a..c5ddb9f021 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { +int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 4096; @@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { char file[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { code = 0; @@ -50,38 +50,69 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { dError("failed to read %s since deployed not found", file); goto _OVER; } - *pDeployed = deployed->valueint; + pOption->deploy = deployed->valueint; - cJSON *id = cJSON_GetObjectItem(root, "id"); - if (id) { - if (id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); + cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex"); + if (selfIndex) { + if (selfIndex->type != cJSON_Number) { + dError("failed to read %s since selfIndex not found", file); goto _OVER; } - if (pReplica) { - pReplica->id = id->valueint; - } + pOption->selfIndex = selfIndex->valueint; } - cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn"); - if (fqdn) { - if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); + cJSON *replicas = cJSON_GetObjectItem(root, "replicas"); + if (replicas) { + if (replicas->type != cJSON_Array) { + dError("failed to read %s since replicas not found", file); goto _OVER; } - if (pReplica) { - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - } - } - cJSON *port = cJSON_GetObjectItem(root, "port"); - if (port) { - if (port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); + int32_t numOfReplicas = cJSON_GetArraySize(replicas); + if (numOfReplicas <= 0) { + dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas); goto _OVER; } - if (pReplica) { - pReplica->port = (uint16_t)port->valueint; + pOption->numOfReplicas = numOfReplicas; + + for (int32_t i = 0; i < numOfReplicas; ++i) { + SReplica *pReplica = pOption->replicas + i; + + cJSON *replica = cJSON_GetArrayItem(replicas, i); + if (replica == NULL) break; + + cJSON *id = cJSON_GetObjectItem(replica, "id"); + if (id) { + if (id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->id = id->valueint; + } + } + + cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn"); + if (fqdn) { + if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto _OVER; + } + if (pReplica) { + tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + } + } + + cJSON *port = cJSON_GetObjectItem(replica, "port"); + if (port) { + if (port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->port = (uint16_t)port->valueint; + } + } } } @@ -92,18 +123,18 @@ _OVER: if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); if (code == 0) { - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy); } terrno = code; return code; } -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) { +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - if (pReplica != NULL && pReplica->id > 0) { - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port); + if (pOption->deploy && pOption->numOfReplicas > 0) { + len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex); + len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n"); + + for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + const SReplica *pReplica = pOption->replicas + i; + if (pReplica != NULL && pReplica->id > 0) { + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + } + if (i < pOption->numOfReplicas - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }],\n"); + } + } } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy); len += snprintf(content + len, maxLen - len, "}\n"); taosWriteFile(pFile, content, len); @@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) return -1; } - dDebug("successed to write %s, deployed:%d", realfile, deployed); + dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy); return 0; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index ec761e6441..fc45dbf15f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - if (createReq.replica != 1) { + SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1}; + memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (createReq.replicas[i].id == pInput->pData->dnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { terrno = TSDB_CODE_INVALID_OPTION; - dGError("failed to create mnode since %s", terrstr()); + dGError("failed to create mnode since %s, selfIndex is -1", terrstr()); return -1; } - bool deployed = true; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - bool deployed = false; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, NULL, deployed) != 0) { + SMnodeOpt option = {.deploy = false}; + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 49207225a5..e7f71ad420 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) { } static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - if (mmReadFile(&mgmt, NULL, required) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pInput->path, &option) != 0) { return -1; } - if (!(*required)) { + if (!option.deploy) { *required = mmDeployRequired(pInput); + } else { + *required = true; } return 0; } static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { - pOption->standby = false; pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica.id = 1; - pOption->replica.port = tsServerPort; - tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->selfIndex = 0; + pOption->numOfReplicas = 1; + pOption->replicas[0].id = 1; + pOption->replicas[0].port = tsServerPort; + tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } -static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { - pOption->standby = false; +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pReplica->id > 0) { - pOption->standby = true; - pOption->replica = *pReplica; - } } static void mmClose(SMnodeMgmt *pMgmt) { @@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - bool deployed = false; - SReplica replica = {0}; - if (mmReadFile(pMgmt, &replica, &deployed) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pMgmt->path, &option) != 0) { dError("failed to read file since %s", terrstr()); mmClose(pMgmt); return -1; } - SMnodeOpt option = {0}; - if (!deployed) { + if (!option.deploy) { dInfo("mnode start to deploy"); pMgmt->pData->dnodeId = 1; mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); - mmBuildOptionForOpen(pMgmt, &replica, &option); + mmBuildOptionForOpen(pMgmt, &option); } pMgmt->pMnode = mndOpen(pMgmt->path, &option); @@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (!deployed || replica.id > 0) { - deployed = true; - if (mmWriteFile(pMgmt, NULL, deployed) != 0) { + if (option.numOfReplicas > 0) { + option.deploy = true; + option.numOfReplicas = 0; + if (mmWriteFile(pMgmt->path, &option) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index d2b071ec61..849d3ef390 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -72,7 +72,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { mndPostProcessQueryMsg(pMsg); } - dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); + dGTrace("msg:%p is freed, code:%s", pMsg, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d07ec7abb0..076826ebc2 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -51,26 +51,14 @@ static int32_t dmInitMonitor() { static bool dmCheckDiskSpace() { osUpdate(); - if (!osDataSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least , quit", - (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osDataSpaceSufficient()) { + dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osLogSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osLogSpaceSufficient()) { + dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osTempSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osTempSpaceSufficient()) { + dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); } return true; } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 71315a39fd..f55e830a44 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -87,12 +87,13 @@ typedef struct { typedef struct { tsem_t syncSem; int64_t sync; - SReplica replica; int32_t errCode; int32_t transId; SRWLatch lock; - int8_t standby; int8_t leaderTransferFinish; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; } SSyncMgmt; typedef struct { @@ -130,11 +131,10 @@ typedef struct SMnode { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); int64_t mndGenerateUid(const char *name, int32_t len); -int32_t mndAcquireRpcRef(SMnode *pMnode); -void mndReleaseRpcRef(SMnode *pMnode); -void mndSetRestore(SMnode *pMnode, bool restored); -void mndSetStop(SMnode *pMnode); -bool mndGetStop(SMnode *pMnode); +void mndSetRestored(SMnode *pMnode, bool restored); +bool mndGetRestored(SMnode *pMnode); +void mndSetStop(SMnode *pMnode); +bool mndGetStop(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index cb9d70d5ee..993efbcd08 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -24,7 +24,7 @@ extern "C" { int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); -bool mndIsMaster(SMnode *pMnode); +bool mndIsLeader(SMnode *pMnode); int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId); void mndSyncStart(SMnode *pMnode); void mndSyncStop(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6670211d78..3b31873857 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true); if (pShow->pIter == NULL) break; if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9ade39e8e7..fba1fd94d6 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { goto _OVER; } else { pDnode->accessTimes++; - mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); + mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } } @@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } pDnode->lastAccessTime = curMs; + pDnode->accessTimes++; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 275fc76e36..5fb23b045c 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -43,6 +43,37 @@ #include "mndUser.h" #include "mndVgroup.h" +static inline int32_t mndAcquireRpc(SMnode *pMnode) { + int32_t code = 0; + taosThreadRwlockRdlock(&pMnode->lock); + if (pMnode->stopped) { + terrno = TSDB_CODE_APP_NOT_READY; + code = -1; + } else if (!mndIsLeader(pMnode)) { + code = -1; + } else { +#if 1 + atomic_add_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is acquired, ref:%d", ref); +#endif + } + taosThreadRwlockUnlock(&pMnode->lock); + return code; +} + +static inline void mndReleaseRpc(SMnode *pMnode) { + taosThreadRwlockRdlock(&pMnode->lock); +#if 1 + atomic_sub_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is released, ref:%d", ref); +#endif + taosThreadRwlockUnlock(&pMnode->lock); +} + static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -201,7 +232,7 @@ static int32_t mndInitWal(SMnode *pMnode) { pMnode->pWal = walOpen(path, &cfg); if (pMnode->pWal == NULL) { - mError("failed to open wal since %s", terrstr()); + mError("failed to open wal since %s. wal:%s", terrstr(), path); return -1; } @@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) { static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->msgCb = pOption->msgCb; pMnode->selfDnodeId = pOption->dnodeId; - pMnode->syncMgmt.replica = pOption->replica; - pMnode->syncMgmt.standby = pOption->standby; + pMnode->syncMgmt.selfIndex = pOption->selfIndex; + pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas; + memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas)); } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) { mError("failed to deploy sdb while start mnode"); return -1; } - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } grantReset(pMnode, TSDB_GRANT_ALL, 0); @@ -570,23 +602,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } - if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; + if (mndAcquireRpc(pMsg->info.node) == 0) return 0; if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { return -1; } - SEpSet epSet = {0}; - mndGetMnodeEpSet(pMsg->info.node, &epSet); + SEpSet epSet = {0}; + SMnode *pMnode = pMsg->info.node; + mndGetMnodeEpSet(pMnode, &epSet); const STraceId *trace = &pMsg->info.traceId; - mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); + mDebug( + "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s " + "numOfEps:%d inUse:%d", + pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync), + syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); if (epSet.numOfEps > 0) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { - mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); } int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); @@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); int32_t code = (*fp)(pMsg); - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mGTrace("msg:%p, won't response immediately since in progress", pMsg); @@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) { int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { - if (mndAcquireRpcRef(pMnode) != 0) return -1; + if (mndAcquireRpc(pMnode) != 0) return -1; SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); @@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc)); if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || pStbInfo->stbs == NULL) { - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return -1; } @@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->timeseries_total = INT32_MAX; } - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return 0; } @@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; } -int32_t mndAcquireRpcRef(SMnode *pMnode) { - int32_t code = 0; - taosThreadRwlockRdlock(&pMnode->lock); - if (pMnode->stopped) { - mTrace("mnode not running"); - terrno = TSDB_CODE_APP_NOT_READY; - code = -1; - } else if (!mndIsMaster(pMnode)) { - mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored); - code = -1; - } else { - int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is acquired, ref:%d", ref); - } - taosThreadRwlockUnlock(&pMnode->lock); - return code; -} - -void mndReleaseRpcRef(SMnode *pMnode) { - taosThreadRwlockRdlock(&pMnode->lock); - int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is released, ref:%d", ref); - taosThreadRwlockUnlock(&pMnode->lock); -} - -void mndSetRestore(SMnode *pMnode, bool restored) { +void mndSetRestored(SMnode *pMnode, bool restored) { if (restored) { taosThreadRwlockWrlock(&pMnode->lock); pMnode->restored = true; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d0440359ff..c27241317c 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -36,6 +36,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq); static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); +static void mndReloadSyncConfig(SMnode *pMnode); int32_t mndInitMnode(SMnode *pMnode) { SSdbTable table = { @@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { } pObj->state = TAOS_SYNC_STATE_ERROR; + mndReloadSyncConfig(pSdb->pMnode); return 0; } @@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; + mndReloadSyncConfig(pSdb->pMnode); + return 0; } @@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pIter == NULL) break; if (pObj->id == pMnode->selfDnodeId) { - if (mndIsMaster(pMnode)) { + if (mndIsLeader(pMnode)) { pEpSet->inUse = pEpSet->numOfEps; } else { pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes; @@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pEpSet->numOfEps == 0) { syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); } + + if (pEpSet->inUse >= pEpSet->numOfEps) { + pEpSet->inUse = 0; + } } static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { @@ -274,13 +282,72 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod return 0; } +static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); + + STransAction action = { + .epSet = *pCreateEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_CREATE_MNODE, + .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + +static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); + + STransAction action = { + .epSet = *pAlterEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_MND_ALTER_MNODE, + .acceptableCode = 0, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { + int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq); + + STransAction action = { + .epSet = *pDroprEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_DROP_MNODE, + .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; SDCreateMnodeReq createReq = {0}; - SEpSet alterEpset = {0}; SEpSet createEpset = {0}; while (1) { @@ -288,75 +355,25 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); if (pIter == NULL) break; - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } + createReq.replicas[numOfReplicas].id = pMObj->id; + createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); numOfReplicas++; sdbRelease(pSdb, pMObj); } - alterReq.replica = numOfReplicas + 1; - alterReq.replicas[numOfReplicas].id = pDnode->id; - alterReq.replicas[numOfReplicas].port = pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.numOfEps = numOfReplicas + 1; - alterEpset.eps[numOfReplicas].port = pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - createReq.replica = 1; - createReq.replicas[0].id = pDnode->id; - createReq.replicas[0].port = pDnode->port; - memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createReq.replica = numOfReplicas + 1; + createReq.replicas[numOfReplicas].id = pDnode->id; + createReq.replicas[numOfReplicas].port = pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createEpset.inUse = 0; createEpset.numOfEps = 1; createEpset.eps[0].port = pDnode->port; memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); - - STransAction action = { - .epSet = createEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_CREATE_MNODE, - .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } + if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; return 0; } @@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); + if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -459,107 +476,28 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO } static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; - SDDropMnodeReq dropReq = {0}; - SSetStandbyReq standbyReq = {0}; - SEpSet alterEpset = {0}; - SEpSet dropEpSet = {0}; - - while (1) { - SMnodeObj *pMObj = NULL; - pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); - if (pIter == NULL) break; - if (pMObj->id == pObj->id) { - sdbRelease(pSdb, pMObj); - continue; - } - - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } - - numOfReplicas++; - sdbRelease(pSdb, pMObj); - } - - alterReq.replica = numOfReplicas; - alterEpset.numOfEps = numOfReplicas; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + int32_t numOfReplicas = 0; + SDDropMnodeReq dropReq = {0}; + SEpSet dropEpSet = {0}; dropReq.dnodeId = pDnode->id; dropEpSet.numOfEps = 1; dropEpSet.eps[0].port = pDnode->port; memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - standbyReq.dnodeId = pDnode->id; - standbyReq.standby = 1; - - { - int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq); - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(MNODE_HANDLE); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_SYNC_SET_MNODE_STANDBY, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_DROP_MNODE, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); + if (totalMnodes == 2) { + mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + } else if (totalMnodes == 3) { + mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes); + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + } else { + return -1; } return 0; @@ -567,7 +505,6 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { if (pObj == NULL) return 0; - if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; if (mndTransAppendNullLog(pTrans) != 0) return -1; @@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { - pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true); if (pShow->pIter == NULL) break; cols = 0; @@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { } static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { +#if 1 + return 0; +#else SMnode *pMnode = pReq->info.node; SDAlterMnodeReq alterReq = {0}; @@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { return -1; } + SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1}; + memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { + mInfo("alter mnode not processed since selfIndex is -1", terrstr()); + return 0; + } + + if (mndWriteFile(pMnode->path, &option) != 0) { + mError("failed to write mnode file since %s", terrstr()); + return -1; + } + SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1}; for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodePort = alterReq.replicas[i].port; - if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i; + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + cfg.myIndex = i; + } } if (cfg.myIndex == -1) { mError("failed to alter mnode since myindex is -1"); return -1; } else { - mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex); + mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); } } - mInfo("trans:-1, sync reconfig will be proposed"); - - SSyncMgmt *pMgmt = &pMnode->syncMgmt; - pMgmt->standby = 0; - int32_t code = syncReconfig(pMgmt->sync, &cfg); + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); if (code != 0) { - mError("trans:-1, failed to propose sync reconfig since %s", terrstr()); - return code; + mError("failed to sync reconfig since %s", terrstr()); } else { - pMgmt->errCode = 0; - taosWLockLatch(&pMgmt->lock); - pMgmt->transId = -1; - taosWUnLockLatch(&pMgmt->lock); - tsem_wait(&pMgmt->syncSem); - mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode)); - terrno = pMgmt->errCode; - return pMgmt->errCode; + mInfo("alter mnode sync success"); + } + + return code; +#endif +} + +static void mndReloadSyncConfig(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + SMnodeObj *pObj = NULL; + ESdbStatus objStatus = 0; + void *pIter = NULL; + bool hasUpdatingMnode = false; + SSyncCfg cfg = {.myIndex = -1}; + + while (1) { + pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false); + if (pIter == NULL) break; + if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) { + mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); + hasUpdatingMnode = true; + } + + if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { + SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; + tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pObj->pDnode->port; + if (pObj->pDnode->id == pMnode->selfDnodeId) { + cfg.myIndex = cfg.replicaNum; + } + cfg.replicaNum++; + } + + sdbReleaseLock(pSdb, pObj, false); + } + + if (cfg.myIndex == -1) { + mInfo("vgId:1, mnode not reload since selfIndex is -1"); + return; + } + + if (!mndGetRestored(pMnode)) { + mInfo("vgId:1, mnode not reload since restore not finished"); + return; + } + + if (hasUpdatingMnode) { + mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); + for (int32_t i = 0; i < cfg.replicaNum; ++i) { + SNodeInfo *pNode = &cfg.nodeInfo[i]; + mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + } + + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); + if (code != 0) { + mError("vgId:1, failed to reconfig mnode sync since %s", terrstr()); + } else { + mInfo("vgId:1, reconfig mnode sync success"); + } } } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 511a84290d..1c8ec0a302 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -416,7 +416,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } SStreamTask* pTask = tNewSStreamTask(pStream->uid); - if (pInnerTask == NULL) { + if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0e4dd9fc3c..866a3fa8b9 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { if (!pMnode->deploy) { mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); mndTransPullup(pMnode); - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } else { mInfo("vgId:1, sync restore finished"); } @@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) { snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); syncInfo.pWal = pMnode->pWal; syncInfo.pFsm = mndSyncMakeFsm(pMnode); - syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT; - mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby); - if (pMgmt->standby || pMgmt->replica.id > 0) { - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = 1; - pCfg->myIndex = 0; - SNodeInfo *pNode = &pCfg->nodeInfo[0]; - tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); - pNode->nodePort = pMgmt->replica.port; - mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort); + mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas); + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = pMgmt->numOfReplicas; + pCfg->myIndex = pMgmt->selfIndex; + for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pMgmt->replicas[i].port; + mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); @@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) { setPingTimerMS(pMgmt->sync, 5000); setElectTimerMS(pMgmt->sync, 3000); setHeartbeatTimerMS(pMgmt->sync, 500); - /* - setElectTimerMS(pMgmt->sync, 600); - setHeartbeatTimerMS(pMgmt->sync, 300); - */ mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; @@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); syncStart(pMgmt->sync); - mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); + mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync); } void mndSyncStop(SMnode *pMnode) { @@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) { taosWUnLockLatch(&pMnode->syncMgmt.lock); } -bool mndIsMaster(SMnode *pMnode) { +bool mndIsLeader(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; if (!syncIsReady(pMgmt->sync)) { @@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) { return false; } - if (!pMnode->restored) { + if (!mndGetRestored(pMnode)) { terrno = TSDB_CODE_APP_NOT_READY; return false; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d30cc8f9ca..a64b07fdbd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -53,7 +53,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); -static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } +static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index f922baf329..be93bd2f59 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey); * @param pObj The object of the row. */ void sdbRelease(SSdb *pSdb, void *pObj); +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock); /** * @brief Traverse a sdb table @@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status); +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock); /** * @brief Cancel a traversal diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index ecdf8c71a7..0fee0fbca1 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { taosThreadRwlockUnlock(pLock); } -void sdbRelease(SSdb *pSdb, void *pObj) { +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { if (pObj == NULL) return; SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); if (pRow->type >= SDB_MAX) return; TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + if (lock) { + taosThreadRwlockWrlock(pLock); + } int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); @@ -342,9 +344,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) { sdbFreeRow(pSdb, pRow, true); } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } } +void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); } + void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { *ppObj = NULL; @@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { return ppRow; } -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) { +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) { *ppObj = NULL; SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + if (lock) { + taosThreadRwlockRdlock(pLock); + } SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { @@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat *status = pRow->status; break; } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } return ppRow; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f96afe6fba..69e6cdce9f 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -115,7 +115,6 @@ typedef struct { typedef struct { SMqDataRsp dataRsp; - SMqRspHead rspHead; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; SRpcHandleInfo pInfo; } STqPushEntry; @@ -183,6 +182,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t ver); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c8a4ff945d..1e0e757793 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema return -1; } - strcpy(pMetaRsp->tbName, tbName); + strncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->tableType = TSDB_NORMAL_TABLE; pMetaRsp->sversion = pSchema->version; @@ -693,6 +693,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; + ASSERT(pAlterTbReq->colName); if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } @@ -816,6 +817,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA const void *pData = NULL; int nData = 0; + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 59a9f32642..5e1cc15063 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -171,11 +171,11 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); - } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); - } + /*if (pRsp->blockNum > 0) {*/ + /*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/ + /*} else {*/ + ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + /*}*/ } int32_t len = 0; @@ -192,7 +192,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { return -1; } - memcpy(buf, &pPushEntry->rspHead, sizeof(SMqRspHead)); + memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -215,7 +215,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vgId:%d, from consumer:%" PRId64 ", (epoch %d) push rsp, block num: %d, reqOffset:%s, rspOffset:%s", - TD_VID(pTq->pVnode), pPushEntry->rspHead.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); + TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); return 0; } @@ -560,9 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); dataRsp.withTbName = 0; memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp)); - pPushEntry->rspHead.consumerId = consumerId; - pPushEntry->rspHead.epoch = reqEpoch; - pPushEntry->rspHead.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + pPushEntry->dataRsp.head.consumerId = consumerId; + pPushEntry->dataRsp.head.epoch = reqEpoch; + pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*)); tqDebug("tmq poll: consumer %ld, subkey %s, vg %d save handle to push mgr", consumerId, pHandle->subKey, TD_VID(pTq->pVnode)); @@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink; + pTask->tbSink.tbSinkFunc = tqTableSink1; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); @@ -1010,16 +1010,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); + + taosFreeQitem(pRefBlock); continue; } + if (streamSchedExec(pTask) < 0) { qError("stream task launch failed, task id %d", pTask->taskId); continue; } + } else { streamTaskInputFail(pTask); } } + int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index bcdac1941a..4083a1a0ae 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -245,7 +245,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tqDebug("vgId:%d cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey); continue; } - if (pPushEntry->dataRsp.reqOffset.version > ver) { + if (pPushEntry->dataRsp.reqOffset.version >= ver) { tqDebug("vgId:%d push entry req version %ld, while push version %ld, skip", pTq->pVnode->config.vgId, pPushEntry->dataRsp.reqOffset.version, ver); continue; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 521d12fdab..76bfa72caa 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,6 +284,212 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; + + int32_t blockSz = taosArrayGetSize(pBlocks); + bool createTb = true; + + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(serializedDeleteReq); + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else { + SVCreateTbReq createTbReq = {0}; + + // set const + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + taosArrayPush(tagArray, &tagVal); + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(tagArray); + tdDestroySVCreateTbReq(&createTbReq); + return; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + createTbReq.ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + createTbReq.name = strdup(pDataBlock->info.parTbName); + } else { + createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + } + + int32_t schemaLen; + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + // save schema str + void* schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + tEncoderClear(&encoder); + taosMemoryFree(schemaStr); + return; + } + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + + int32_t cap = sizeof(SSubmitReq); + + int32_t rows = pDataBlock->info.rows; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + + cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; + + SSubmitReq* ret = rpcMallocCont(cap); + ret->header.vgId = pVnode->config.vgId; + ret->length = sizeof(SSubmitReq); + ret->numOfBlocks = htonl(1); + + SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + + blkHead->numOfRows = htonl(pDataBlock->info.rows); + blkHead->sversion = htonl(pTSchema->version); + blkHead->suid = htobe64(suid); + // uid is assigned by vnode + blkHead->uid = 0; + blkHead->schemaLen = 0; + + tqDebug("tq sink, convert block %d, rows: %d", i, rows); + + int32_t dataLen = 0; + void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); + STSRow* rowData = blkSchema; + if (createTb) { + memcpy(blkSchema, schemaStr, schemaLen); + blkHead->schemaLen = htonl(schemaLen); + rowData = POINTER_SHIFT(blkSchema, schemaLen); + } + + taosMemoryFree(schemaStr); + + for (int32_t j = 0; j < rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); + } else { + void* colData = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); + } + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + blkHead->dataLen = htonl(dataLen); + + ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + ret->length = htonl(ret->length); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = ret, + .contLen = ntohl(ret->length), + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(ret); + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } + taosArrayDestroy(tagArray); +} + void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ac6be9af2d..0ee2e31097 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD if (code) goto _err; } +_err: if (aDelData) { taosArrayDestroy(aDelData); } -_err: return code; } @@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); - if (code) goto _err; *pDelIdx = *pIdx; +_err: if (pDelIdxArray) { taosArrayDestroy(pDelIdxArray); } -_err: return code; } @@ -429,7 +428,8 @@ typedef struct { SDataFReader *pDataFReader; TSDBROW row; - SMergeTree mergeTree; + SMergeTree mergeTree; + SMergeTree *pMergeTree; } SFSLastNextRowIter; static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { @@ -444,11 +444,14 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { case SFSLASTNEXTROW_FILESET: { SDFileSet *pFileSet = NULL; _next_fileset: + if (state->pMergeTree != NULL) { + tMergeTreeClose(state->pMergeTree); + state->pMergeTree = NULL; + } + if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - tMergeTreeClose(&state->mergeTree); - *ppRow = NULL; return code; } @@ -460,10 +463,10 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, pLoadInfo, true, NULL); + state->pMergeTree = &state->mergeTree; bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; - tMergeTreeClose(&state->mergeTree); goto _next_fileset; } state->state = SFSLASTNEXTROW_BLOCKROW; @@ -475,6 +478,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; } + return code; default: ASSERT(0); @@ -486,6 +490,11 @@ _err: tsdbDataFReaderClose(&state->pDataFReader); state->pDataFReader = NULL; } + if (state->pMergeTree != NULL) { + tMergeTreeClose(state->pMergeTree); + state->pMergeTree = NULL; + } + *ppRow = NULL; return code; @@ -504,6 +513,11 @@ int32_t clearNextRowFromFSLast(void *iter) { state->pDataFReader = NULL; } + if (state->pMergeTree != NULL) { + tMergeTreeClose(state->pMergeTree); + state->pMergeTree = NULL; + } + return code; } @@ -584,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (code) goto _err; - if (!state->pBlockIdx) { goto _next_fileset; } @@ -883,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs if (code) goto _err; code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } tsdbDelFReaderClose(&pDelFReader); } else { @@ -1216,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { _err: nextRowIterClose(&iter); taosMemoryFreeClear(pTSchema); + *ppLastArray = NULL; + taosArrayDestroy(pColArray); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 995d7a3155..a4d993bc6c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -396,12 +396,19 @@ _exit: return code; } +static int32_t tDataIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + SDataIter *pIter1 = (SDataIter *)((uint8_t *)n1 - offsetof(SDataIter, n)); + SDataIter *pIter2 = (SDataIter *)((uint8_t *)n2 - offsetof(SDataIter, n)); + + return tRowInfoCmprFn(&pIter1->r, &pIter2->r); +} + static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; int32_t lino = 0; pCommitter->pIter = NULL; - tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pCommitter->rbt, tDataIterCmprFn); // memory TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; @@ -1038,7 +1045,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - ASSERT(eno == 0); + ASSERT(eno == 0 && + "tsdbCommit failure" + "Restart taosd"); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 85514ed5b6..6fd5629592 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -610,9 +610,6 @@ int32_t tsdbFSRollback(STsdbFS *pFS) { ASSERT(0); return code; - -_err: - return code; } int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) { @@ -822,7 +819,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pHeadF); } } else { @@ -866,7 +863,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); if (nRef == 0) { tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pSmaF); } } else { @@ -877,7 +874,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { // stt if (sameDisk) { if (pSetNew->nSttF > pSetOld->nSttF) { - ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + ASSERT(pSetNew->nSttF == pSetOld->nSttF + 1); pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -964,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSetOld->pHeadF); } @@ -1104,7 +1101,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbDelFileName(pTsdb, pFS->pDelFile, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pFS->pDelFile); } } @@ -1117,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSet->pHeadF); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index a6628463f8..0189ced3c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid goto _err; } + ASSERT(pPool != NULL); // do delete SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData)); if (pDelData == NULL) { @@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; + ASSERT(pPool != NULL); pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN // node level = tsdbMemSkipListRandLevel(&pTbData->sl); + ASSERT(pPool != NULL); pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -559,6 +562,8 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i // backward put first data row.pTSRow = tGetSubmitBlkNext(&blkIter); + if (row.pTSRow == NULL) return code; + key.ts = row.pTSRow->ts; nRow++; tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 4d54fbbd79..2f4fdfc5f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -101,8 +101,8 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { int32_t code = 0; - SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo; - if (pInfo->blockIndex[0] == pIter->iSttBlk) { + SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo; + if (pInfo->blockIndex[0] == pIter->iSttBlk) { if (pInfo->currentLoadBlockIndex != 0) { tsdbDebug("current load index is set to 0, block index:%d, file index:%d, due to uid:%" PRIu64 ", load data, %s", pIter->iSttBlk, pIter->iStt, pIter->uid, idStr); @@ -113,7 +113,7 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { if (pInfo->blockIndex[1] == pIter->iSttBlk) { if (pInfo->currentLoadBlockIndex != 1) { - tsdbDebug("current load index is set to 1, block index:%d, file index:%d, due to uid:%"PRIu64", load data, %s", + tsdbDebug("current load index is set to 1, block index:%d, file index:%d, due to uid:%" PRIu64 ", load data, %s", pIter->iSttBlk, pIter->iStt, pIter->uid, idStr); pInfo->currentLoadBlockIndex = 1; } @@ -133,18 +133,24 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { id.uid = pIter->uid; } - tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } double el = (taosGetTimestampUs() - st) / 1000.0; pInfo->elapsedTime += el; pInfo->loadBlocks += 1; - tsdbDebug("read last block, total load:%d, trigger by uid:%"PRIu64", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", - pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + tsdbDebug("read last block, total load:%d, trigger by uid:%" PRIu64 + ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", + pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, + idStr); pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr); @@ -336,7 +342,7 @@ _exit: void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); } -void tLDataIterNextBlock(SLDataIter *pIter, const char* idStr) { +void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { int32_t step = pIter->backward ? -1 : 1; int32_t oldIndex = pIter->iSttBlk; @@ -386,10 +392,10 @@ void tLDataIterNextBlock(SLDataIter *pIter, const char* idStr) { if (index != -1) { pIter->iSttBlk = index; pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); - tsdbDebug("try next last file block:%d from %d, trigger by uid:%"PRIu64", file index:%d, %s", pIter->iSttBlk, oldIndex, pIter->uid, pIter->iStt, - idStr); + tsdbDebug("try next last file block:%d from %d, trigger by uid:%" PRIu64 ", file index:%d, %s", pIter->iSttBlk, + oldIndex, pIter->uid, pIter->iStt, idStr); } else { - tsdbDebug("no more last block qualified, uid:%"PRIu64", file index::%d, %s", pIter->uid, oldIndex, idStr); + tsdbDebug("no more last block qualified, uid:%" PRIu64 ", file index::%d, %s", pIter->uid, oldIndex, idStr); } } @@ -457,8 +463,8 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { - int32_t code = 0; int32_t step = pIter->backward ? -1 : 1; + terrno = TSDB_CODE_SUCCESS; // no qualified last file block in current file, no need to fetch row if (pIter->pSttBlk == NULL) { @@ -467,6 +473,10 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { int32_t iBlockL = pIter->iSttBlk; SBlockData *pBlockData = loadLastBlock(pIter, idStr); + if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) { + goto _exit; + } + pIter->iRow += step; while (1) { @@ -492,19 +502,15 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - } - - return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); + return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } // SMergeTree ================================================= -static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { - SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode)); - SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) { + SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node)); + SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node)); TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); @@ -541,6 +547,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; + ASSERT(pMTree->pLoadInfo != NULL); for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter *pIter = NULL; @@ -581,7 +588,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node)); + int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index e4080ccf1e..5197823490 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -87,9 +87,13 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { - taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + taosThreadRwlockWrlock(&(*pTsdb)->rwLock); tsdbMemTableDestroy((*pTsdb)->mem); (*pTsdb)->mem = NULL; + taosThreadRwlockUnlock(&(*pTsdb)->rwLock); + + taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + tsdbFSClose(*pTsdb); tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4bb1f2d2fe..670691ab1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -340,7 +340,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); if (pIter->pLastBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); + tsdbError("failed to prepare the last block iterator, code:%s %s", tstrerror(code), pReader->idStr); return code; } } @@ -646,7 +646,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug( - "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "load block of %"PRIzu" tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); @@ -1515,6 +1515,11 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* taosMemoryFree(pReader->pMemSchema); int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return pReader->pMemSchema; } @@ -1645,7 +1650,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STSRow* pTSRow = NULL; SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbTrace("fRow ptr:%p, %d, uid:%"PRIu64", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { @@ -2676,9 +2681,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret int8_t level = 0; int8_t precision = pVnode->config.tsdbCfg.precision; int64_t now = taosGetTimestamp(precision); - int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1 - : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000 - : 1000000); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L + : 1000000L); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; @@ -3085,7 +3090,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } int32_t code = tRowMergerGetRow(&merge, pTSRow); @@ -3372,14 +3377,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl if (pCond->suid != 0) { pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1, - pReader->idStr); + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr); } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr); + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); } } @@ -3395,19 +3399,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - code = doOpenReaderImpl(pReader); + if (numOfTables > 0) { + code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { - return code; + goto _err; } - } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - STsdbReader* pNextReader = pReader->innerReader[1]; + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; // we need only one row pPrevReader->capacity = 1; @@ -3422,19 +3427,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl pNextReader->pMemSchema = pReader->pMemSchema; pNextReader->pReadSnap = pReader->pReadSnap; - code = doOpenReaderImpl(pPrevReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pNextReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pNextReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } @@ -3442,7 +3448,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl return code; _err: - tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); return code; } @@ -3513,6 +3519,10 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pLReader); } + if (pReader->innerReader[0] != 0) { + tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr); + } + tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 " SMA-time:%.2f ms, fileBlocks:%" PRId64 @@ -3727,7 +3737,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { - if (isEmptyQueryTimeWindow(&pReader->window)) { + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 167b4a104c..84b3afe72c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -18,7 +18,7 @@ // =============== PAGE-WISE FILE =============== static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; - STsdbFD *pFD; + STsdbFD *pFD = NULL; *ppFD = NULL; @@ -35,6 +35,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pFD = taosOpenFile(path, flag); if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD); goto _exit; } pFD->szPage = szPage; @@ -42,11 +43,15 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFD->pFD); taosMemoryFree(pFD); goto _exit; } if (taosStatFile(path, &pFD->szFile, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); goto _exit; } ASSERT(pFD->szFile % szPage == 0); @@ -59,10 +64,12 @@ _exit: static void tsdbCloseFile(STsdbFD **ppFD) { STsdbFD *pFD = *ppFD; - taosMemoryFree(pFD->pBuf); - taosCloseFile(&pFD->pFD); - taosMemoryFree(pFD); - *ppFD = NULL; + if (pFD) { + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); + *ppFD = NULL; + } } static int32_t tsdbWriteFilePage(STsdbFD *pFD) { @@ -117,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { } // check - if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } @@ -443,7 +450,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p pBlockIdx->size = size; pHeadFile->size += size; - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 " size:%" PRId64 " nItem:%d", TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); @@ -457,7 +464,7 @@ _err: int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { int32_t code = 0; SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; - int64_t size; + int64_t size = 0; int64_t n; // check @@ -836,7 +843,8 @@ _err: // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; - SDataFReader *pReader; + int32_t lino = 0; + SDataFReader *pReader = NULL; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fname[TSDB_FILENAME_LEN]; @@ -844,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->pSet = pSet; @@ -852,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // head tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // data tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // sma tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // stt for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppReader = pReader; - return code; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); -_err: - tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; + if (pReader) { + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]); + tsdbCloseFile(&pReader->pSmaFD); + tsdbCloseFile(&pReader->pDataFD); + tsdbCloseFile(&pReader->pHeadFD); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + } return code; } @@ -906,10 +923,6 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { taosMemoryFree(*ppReader); *ppReader = NULL; return code; - -_err: - tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code)); - return code; } int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { @@ -1289,16 +1302,17 @@ _exit: // SDelFWriter ==================================================== int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter; + SDelFWriter *pDelFWriter = NULL; int64_t n; // alloc pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); if (pDelFWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pDelFWriter->pTsdb = pTsdb; pDelFWriter->fDel = *pFile; @@ -1306,21 +1320,28 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // update header code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pDelFWriter->fDel.size = TSDB_FHDR_SIZE; pDelFWriter->fDel.offset = 0; *ppWriter = pDelFWriter; - return code; -_err: - tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; +_exit: + if (code) { + if (pDelFWriter) { + taosMemoryFree(pDelFWriter); + tsdbCloseFile(&pDelFWriter->pWriteH); + } + *ppWriter = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); + } else { + *ppWriter = pDelFWriter; + } return code; } @@ -1456,15 +1477,15 @@ struct SDelFReader { int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; - SDelFReader *pDelFReader; - int64_t n; + SDelFReader *pDelFReader = NULL; // alloc pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); if (pDelFReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } // open impl @@ -1473,14 +1494,18 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH); - if (code) goto _err; + if (code) { + taosMemoryFree(pDelFReader); + goto _exit; + } - *ppReader = pDelFReader; - return code; - -_err: - tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + *ppReader = pDelFReader; + } return code; } @@ -1573,4 +1598,4 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a928dc3484..74c704598d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -67,6 +67,13 @@ extern int32_t tRowInfoCmprFn(const void* p1, const void* p2); extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData); extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); +static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) { + SFDataIter* pIter1 = (SFDataIter*)(((uint8_t*)pNode1) - offsetof(SFDataIter, n)); + SFDataIter* pIter2 = (SFDataIter*)(((uint8_t*)pNode2) - offsetof(SFDataIter, n)); + + return tRowInfoCmprFn(&pIter1->rInfo, &pIter2->rInfo); +} + static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { int32_t code = 0; @@ -79,7 +86,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (code) goto _err; pReader->pIter = NULL; - tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn); // .data file SFDataIter* pIter = &pReader->aFDataIter[0]; @@ -421,7 +428,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { n += tPutDelData((*ppData) + n, pDelData); } - tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%d" PRId64 " size:%d", + tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%" PRId64 " size:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pDelIdx->suid, pDelIdx->uid, size); break; @@ -431,20 +438,21 @@ _exit: return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->pVnode, + tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); return code; } int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { int32_t code = 0; + int32_t lino = 0; STsdbSnapReader* pReader = NULL; // alloc pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->sver = sver; @@ -454,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = taosThreadRwlockRdlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFSRef(pTsdb, &pReader->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = taosThreadRwlockUnlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // data @@ -478,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pIter->aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } else { pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tBlockDataCreate(&pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pReader->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // del pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); if (pReader->aDelIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pReader->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); - *ppReader = pReader; - return code; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code), + pTsdb->path); + *ppReader = NULL; -_err: - tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppReader = NULL; + if (pReader) { + taosArrayDestroy(pReader->aDelData); + taosArrayDestroy(pReader->aDelIdx); + tBlockDataDestroy(&pReader->bData, 1); + tsdbFSDestroy(&pReader->fs); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); + } return code; } @@ -1247,20 +1264,21 @@ _err: // APIs int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { int32_t code = 0; + int32_t lino = 0; STsdbSnapWriter* pWriter = NULL; // alloc pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->pTsdb = pTsdb; pWriter->sver = sver; pWriter->ever = ever; code = tsdbFSCopy(pTsdb, &pWriter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // config pWriter->minutes = pTsdb->keepCfg.days; @@ -1272,7 +1290,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr // SNAP_DATA_TSDB code = tBlockDataCreate(&pWriter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pWriter->fid = INT32_MIN; pWriter->id = (TABLEID){0}; @@ -1280,53 +1298,67 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dReader.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // Writer pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pWriter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dWriter.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataCreate(&pWriter->dWriter.sData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // SNAP_DATA_DEL pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pWriter->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppWriter = pWriter; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + *ppWriter = NULL; - tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppWriter = NULL; + if (pWriter) { + if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW); + if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData); + if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR); + tBlockDataDestroy(&pWriter->dWriter.sData, 1); + tBlockDataDestroy(&pWriter->dWriter.bData, 1); + if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk); + if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx); + tBlockDataDestroy(&pWriter->dReader.bData, 1); + if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx); + tBlockDataDestroy(&pWriter->bData, 1); + tsdbFSDestroy(&pWriter->fs); + taosMemoryFree(pWriter); + } + } else { + tsdbDebug("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); + *ppWriter = pWriter; + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 0a9fbf92a4..f38802aee7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -34,7 +34,9 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - tInitSubmitMsgIter(pMsg, &msgIter); + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return -1; + } while (true) { SSubmitBlkRsp r = {0}; tGetSubmitMsgNext(&msgIter, &pBlock); diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 3c0a041197..682f2b4225 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) { void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p = NULL; + ASSERT(pPool != NULL); + taosThreadSpinLock(&pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node @@ -141,17 +143,17 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { } void vnodeBufPoolFree(SVBufPool *pPool, void *p) { - uint8_t *ptr = (uint8_t *)p; - SVBufPoolNode *pNode; + // uint8_t *ptr = (uint8_t *)p; + // SVBufPoolNode *pNode; - if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { - pNode = &((SVBufPoolNode *)p)[-1]; - *pNode->pnext = pNode->prev; - pNode->prev->pnext = pNode->pnext; + // if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { + // pNode = &((SVBufPoolNode *)p)[-1]; + // *pNode->pnext = pNode->prev; + // pNode->prev->pnext = pNode->pnext; - pPool->size = pPool->size - sizeof(*pNode) - pNode->size; - taosMemoryFree(pNode); - } + // pPool->size = pPool->size - sizeof(*pNode) - pNode->size; + // taosMemoryFree(pNode); + // } } void vnodeBufPoolRef(SVBufPool *pPool) { diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 07d9b96261..07c4c32955 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) { int vnodeShouldCommit(SVnode *pVnode) { if (pVnode->inUse) { - return pVnode->inUse->size > pVnode->inUse->node.size; + return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size); } return false; } @@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { data = NULL; if (vnodeEncodeInfo(pInfo, &data) < 0) { + vError("failed to encode json info."); return -1; } @@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } if (taosWriteFile(pFile, data, strlen(data)) < 0) { - vError("failed to write info file:%s data:%s", fname, terrstr()); + vError("failed to write info file:%s error:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } if (vnodeSaveInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit // smaSyncPreCommit(pVnode->pSma); - if (smaAsyncPreCommit(pVnode->pSma) < 0) { - ASSERT(0); + if(smaAsyncPreCommit(pVnode->pSma) < 0){ + vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) { // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (VND_IS_RSMA(pVnode)) { if (smaAsyncCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } else { if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } if (tqCommit(pVnode->pTq) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // walCommit (TODO) // commit info if (vnodeCommitInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) { // postCommit // smaSyncPostCommit(pVnode->pSma); if (smaAsyncPostCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 314c0e7390..9fe37505ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -46,11 +46,17 @@ int vnodeInit(int nthreads) { return 0; } - vnodeGlobal.stop = 0; + taosThreadMutexInit(&vnodeGlobal.mutex, NULL); + taosThreadCondInit(&vnodeGlobal.hasTask, NULL); + taosThreadMutexLock(&vnodeGlobal.mutex); + + vnodeGlobal.stop = 0; vnodeGlobal.queue.next = &vnodeGlobal.queue; vnodeGlobal.queue.prev = &vnodeGlobal.queue; + taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + vnodeGlobal.nthreads = nthreads; vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); if (vnodeGlobal.threads == NULL) { @@ -59,9 +65,6 @@ int vnodeInit(int nthreads) { return -1; } - taosThreadMutexInit(&vnodeGlobal.mutex, NULL); - taosThreadCondInit(&vnodeGlobal.hasTask, NULL); - for (int i = 0; i < nthreads; i++) { taosThreadCreate(&(vnodeGlobal.threads[i]), NULL, loop, NULL); } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index de3af7cde6..001bb5f7f2 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -38,7 +38,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { if (taosMkDir(path)) { return TAOS_SYSTEM_ERROR(errno); } - strcpy(dir, path); + snprintf(dir, TSDB_FILENAME_LEN, "%s", path); } if (pCfg) { @@ -51,7 +51,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.commitID = 0; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { - vError("vgId:%d, failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); return -1; } @@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 08c3a34699..9cbb1b8af4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -166,7 +166,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (*ppData) { goto _exit; } else { - pReader->tqHandleDone = 1; + pReader->tqOffsetDone = 1; code = tqOffsetReaderClose(&pReader->pTqOffsetReader); if (code) goto _err; } @@ -219,7 +219,7 @@ _exit: return code; _err: - vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); return code; } @@ -260,7 +260,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr // commit it code = vnodeCommit(pVnode); - if (code) goto _err; + if (code) { + taosMemoryFree(pWriter); + goto _err; + } // inc commit ID pVnode->state.commitID++; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 131327976f..3556b47dd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -41,16 +41,28 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t nReqs; tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } - tDecodeI32v(&dc, &nReqs); + if (tDecodeI32v(&dc, &nReqs) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } for (int32_t iReq = 0; iReq < nReqs; iReq++) { tb_uid_t uid = tGenIdPI64(); char *name = NULL; tStartDecode(&dc); - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; @@ -68,7 +80,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int64_t ctime = taosGetTimestampMs(); tb_uid_t uid; - tInitSubmitMsgIter(pSubmitReq, &msgIter); + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + code = terrno; + goto _err; + } for (;;) { tGetSubmitMsgNext(&msgIter, &pBlock); @@ -78,10 +93,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { char *name = NULL; tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } uid = metaGetTableEntryUidByName(pVnode->pMeta, name); if (uid == 0) { @@ -145,6 +169,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t len; int32_t ret; + if (!pVnode->inUse) { + terrno = TSDB_CODE_VND_NOT_SYNCED; + vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); + return -1; + } + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); @@ -265,10 +295,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change - vnodeCommit(pVnode); + if (vnodeCommit(pVnode) < 0) { + vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } // start a new one - vnodeBegin(pVnode); + if (vnodeBegin(pVnode) < 0) { + vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } return 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 7b8c66e368..744d6eab06 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -638,7 +638,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_QNODE_LIST; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -692,7 +692,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_DNODE_LIST; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get dnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -743,7 +743,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU int32_t msgLen = 0; int32_t reqType = TDMT_MND_USE_DB; SCtgTask* pTask = tReq ? tReq->pTask : NULL; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -795,7 +795,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_DB_CFG; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); @@ -850,7 +850,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_INDEX; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get index from mnode, indexName:%s", indexName); @@ -905,7 +905,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(name, tbFName); @@ -962,7 +962,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_RETRIEVE_FUNC; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get udf info from mnode, funcName:%s", funcName); @@ -1017,7 +1017,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_USER_AUTH; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get user auth from mnode, user:%s", user); @@ -1077,7 +1077,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* int32_t reqType = TDMT_MND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFName, "%s.%s", dbFName, tbName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); @@ -1140,7 +1140,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa int32_t reqType = TDMT_VND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; ctgDebug("try to get table meta from vnode, vgId:%d, ep num:%d, ep %s:%d, tbFName:%s", vgroupInfo->vgId, @@ -1209,7 +1209,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t reqType = TDMT_VND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1274,7 +1274,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t reqType = TDMT_MND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1326,7 +1326,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_SERVER_VERSION; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; qDebug("try to get svr ver from mnode"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f9e47f92fe..fc996a6003 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -212,6 +212,7 @@ typedef struct SExprSupp { int32_t numOfExprs; // the number of scalar expression in group operator SqlFunctionCtx* pCtx; int32_t* rowEntryInfoOffset; // offset value for each row result cell info + SFilterInfo* pFilterInfo; } SExprSupp; typedef struct SOperatorInfo { @@ -479,6 +480,7 @@ typedef struct SStreamScanInfo { SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; int32_t primaryTsIndex; // primary time stamp slot id SReadHandle readHandle; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. @@ -775,6 +777,7 @@ typedef struct SStreamPartitionOperatorInfo { SPartitionBySupporter partitionSup; SExprSupp scalarSup; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; SHashObj* pPartitions; void* parIte; SSDataBlock* pInputDataBlock; @@ -924,7 +927,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6a43ea40c6..ce0ff485ab 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); tFreeSSubmitRsp(pInserter->submitRes.pRsp); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2f6b737f79..f99c832381 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1138,7 +1138,7 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i s.bytes = bytes; s.slotId = slotId; s.precision = precision; - strncpy(s.name, name, tListLen(s.name)); + tstrncpy(s.name, name, tListLen(s.name)); return s; } @@ -1201,14 +1201,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; + tExprNode* pExprNode = pExp->pExpr; + + pExprNode->_function.functionId = pFuncNode->funcId; + pExprNode->_function.pFunctNode = pFuncNode; + + tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); #if 1 // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + const char* name = "tbname"; + int32_t len = strlen(name); + + if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) && + pExprNode->_function.functionName[len] == 0) { + pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); @@ -1250,6 +1257,17 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pOpNode->node.aliasName); pExp->pExpr->_optrRoot.pRootNode = pNode; + } else if (type == QUERY_NODE_CASE_WHEN) { + pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; + SCaseWhenNode* pCaseNode = (SCaseWhenNode*)pNode; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pCaseNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, + pType->precision, pCaseNode->node.aliasName); + pExp->pExpr->_optrRoot.pRootNode = pNode; } else { ASSERT(0); } @@ -1348,7 +1366,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; - strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e527f7e7c1..41f861cd38 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -616,7 +616,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { int32_t nOptrWithVal = 0; int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { taosMemoryFreeClear(*pOutput); *len = 0; } @@ -701,10 +701,10 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { return 0; } -int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo mtInfo) { +int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { memset(pCond, 0, sizeof(SQueryTableDataCond)); pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = mtInfo.schema->nCols; + pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); if (pCond->colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -712,15 +712,15 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s } pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = mtInfo.suid; + pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = mtInfo.schema->pSchema[i].type; - pCond->colList[i].bytes = mtInfo.schema->pSchema[i].bytes; - pCond->colList[i].colId = mtInfo.schema->pSchema[i].colId; + pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; + pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; + pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; } return TSDB_CODE_SUCCESS; @@ -844,7 +844,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList); if (mtInfo.uid == 0) return 0; // no data - initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo); + initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0}); @@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; - qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts); + qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid); + qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); return -1; } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid); + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; tsdbReaderClose(pInfo->dataReader); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8c1104f519..aaef411d2d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -895,33 +895,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } #endif -static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { -#if 0 - SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; - uint32_t status = BLK_DATA_NOT_LOAD; - - int32_t numOfOutput = 0; // pTableScanInfo->numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId; - - // group by + first/last should not apply the first/last block filter - if (functionId < 0) { - status |= BLK_DATA_DATA_LOAD; - return status; - } else { - // status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); - // if ((status & BLK_DATA_DATA_LOAD) == BLK_DATA_DATA_LOAD) { - // return status; - // } - } - } - - return status; -#endif - return 0; -} - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NOT_LOAD; @@ -1113,15 +1086,24 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo) { if (pFilterNode == NULL || pBlock->info.rows == 0) { return; } - SFilterInfo* filter = NULL; + SFilterInfo* filter = pFilterInfo; + int64_t st = taosGetTimestampUs(); + +// pError("start filter"); // todo move to the initialization function - int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + int32_t code = 0; + bool needFree = false; + if (filter == NULL) { + needFree = true; + code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + } + SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); @@ -1130,7 +1112,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM // todo the keep seems never to be True?? bool keep = filterExecute(filter, pBlock, &p, NULL, param1.numOfCols, &status); - filterFreeInfo(filter); + + if (needFree) { + filterFreeInfo(filter); + } extractQualifiedTupleByFilterResult(pBlock, p, keep, status); @@ -1790,7 +1775,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); + qDebug("%s fetch rsp received, index:%d, code:%s", pSourceDataInfo->taskId, index, tstrerror(code)); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -1845,6 +1830,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); return pTaskInfo->code; } @@ -2479,7 +2465,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pAggInfo->pCondition, pInfo->pRes, NULL); + doFilter(pAggInfo->pCondition, pInfo->pRes, NULL, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -2803,6 +2789,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); blockDataCleanup(pInfo->pRes); + blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); + blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) { @@ -2873,7 +2861,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { break; } - doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo, NULL); if (fillResult->info.rows > 0) { break; } @@ -3049,6 +3037,12 @@ void cleanupExprSupp(SExprSupp* pSupp) { destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); taosMemoryFreeClear(pSupp->pExprInfo); } + + if (pSupp->pFilterInfo != NULL) { + filterFreeInfo(pSupp->pFilterInfo); + pSupp->pFilterInfo = NULL; + } + taosMemoryFree(pSupp->rowEntryInfoOffset); } @@ -3378,8 +3372,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; - strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } // this the tags and pseudo function columns, we only keep the tag columns @@ -3393,7 +3387,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; + pSchema->bytes = pColNode->node.resType.bytes; strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f3ae652987..e62f72c8f8 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -311,7 +311,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes, NULL); + doFilter(pInfo->pCondition, pRes, NULL, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -991,6 +991,8 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFree(pInfo->partitionSup.keyBuf); cleanupExprSupp(&pInfo->scalarSup); + cleanupExprSupp(&pInfo->tbnameCalSup); + cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); taosMemoryFreeClear(param); } @@ -1037,6 +1039,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + if (pPartNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + int32_t keyLen = 0; code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index b4aa6de0a8..53cfa6c27a 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -387,7 +387,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { break; } if (pJoinInfo->pCondAfterMerge != NULL) { - doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL); + doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL, NULL); } if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index e9e6fed66a..fe6ee4129a 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -315,7 +315,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } // do apply filter - doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL); + doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL, NULL); // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { @@ -325,7 +325,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } else { // do apply filter if (pRes->info.rows > 0) { - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); if (pRes->info.rows == 0) { continue; } @@ -518,7 +518,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { } } - doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); + doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL, NULL); size_t rows = pInfo->pRes->info.rows; if (rows > 0 || pOperator->status == OP_EXEC_DONE) { break; @@ -620,7 +620,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { } pRes->info.rows = 1; - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); /*int32_t status = */ doIngroupLimitOffset(&pProjectInfo->limitInfo, 0, pRes, pOperator); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c20bb95811..15a7abf1ff 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -284,6 +284,19 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } + +static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + SExprSupp* pSup = &pTableScanInfo->pseudoSup; + + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, + GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + } +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -312,8 +325,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; @@ -322,6 +336,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -371,21 +386,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); - - // currently only the tbname pseudo column - if (pTableScanInfo->pseudoSup.numOfExprs > 0) { - SExprSupp* pSup = &pTableScanInfo->pseudoSup; - - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, - GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - } + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampUs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, pOperator->exprSupp.pFilterInfo); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -754,6 +759,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; + + if (pInfo->pFilterNode != NULL) { + code = filterInitFromNode((SNode*)pInfo->pFilterNode, &pOperator->exprSupp.pFilterInfo, 0); + } + pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; pInfo->currentGroupId = -1; @@ -1074,12 +1084,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; uint64_t groupId = gpIdCol[*pRowIndex]; + while (1) { if (hasGroup) { (*pRowIndex) += 1; @@ -1123,7 +1134,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 return NULL; } - doFilter(pInfo->pCondition, pResult, NULL); + doFilter(pInfo->pCondition, pResult, NULL, NULL); if (pResult->info.rows == 0) { continue; } @@ -1143,6 +1154,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 pResult->info.rows++; } } + + blockDataDestroy(tmpBlock); + if (pResult->info.rows > 0) { pResult->info.calWin = pInfo->updateWin; return pResult; @@ -1315,6 +1329,11 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* ASSERT(pResBlock->info.rows == 1); // build tagArray + /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/ + /*STagVal tagVal = {*/ + /*.cid = 0,*/ + /*.type = 0,*/ + /*};*/ // build STag // set STag @@ -1469,7 +1488,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } if (filter) { - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); } blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); @@ -1533,7 +1552,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { tsdbReaderClose(pTSInfo->dataReader); pTSInfo->dataReader = NULL; tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1); + qDebug("queue scan tsdb over, switch to wal ver %"PRId64, pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { return NULL; } @@ -1891,7 +1910,7 @@ FETCH_NEXT_BLOCK: } } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { @@ -2164,6 +2183,19 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } + if (pTableScanNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2361,7 +2393,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } @@ -2542,16 +2574,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SMetaReader smrChildTable = {0}; metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smrChildTable, condTableName); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { metaReaderClear(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } + SMetaReader smrSuperTable = {0}; metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); metaReaderClear(&smrSuperTable); metaReaderClear(&smrChildTable); @@ -2958,7 +3001,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); - strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); strcpy(pInfo->req.user, pInfo->pUser); int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); @@ -3434,7 +3477,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, NULL); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -3769,6 +3812,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pColList); goto _error; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index ed3247e10e..0d5f521034 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { return NULL; } - doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo); + doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo, NULL); if (blockDataGetNumOfRows(pBlock) == 0) { continue; } @@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 5be9918185..8b716531e5 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1501,7 +1501,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } doStreamFillImpl(pOperator); - doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo, NULL); pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; if (pInfo->pRes->info.rows > 0) { break; @@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); pInfo->pCondition = pPhyFillNode->node.pConditions; pInfo->pColMatchColInfo = pColMatchColInfo; - initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pInfo->srcRowIndex = 0; pOperator->name = "FillOperator"; @@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi _error: destroyStreamFillOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index df16914cb8..cbbd874bb9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1277,7 +1277,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -1315,7 +1315,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBlock, NULL); + doFilter(pInfo->pCondition, pBlock, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; + STimeWindow win = {0}; if (IS_FINAL_OP(pInfo)) { win.skey = startTsCols[i]; @@ -2019,7 +2020,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -2062,7 +2063,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -2841,14 +2842,13 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->twAggSup.waterMark = pSessionNode->window.watermark; pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType; pInfo->gap = pSessionNode->gap; @@ -4665,7 +4665,6 @@ void destroyStreamStateOperatorInfo(void* param) { SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo); taosMemoryFreeClear(pChild); - taosMemoryFreeClear(pChInfo); } } colDataDestroy(&pInfo->twAggSup.timeWindowData); @@ -5251,7 +5250,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes); - doFilter(pMiaInfo->pCondition, pRes, NULL); + doFilter(pMiaInfo->pCondition, pRes, NULL, NULL); if (pRes->info.rows >= pOperator->resultInfo.capacity) { break; } @@ -5828,27 +5827,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->isFinal = false; - if (pIntervalPhyNode->window.pExprs != NULL) { - int32_t numOfScalar = 0; - SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; + + initBasicInfo(&pInfo->binfo, pResBlock); + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); pInfo->invertible = false; diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 16fd11f97d..134862d88c 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -18,7 +18,7 @@ #include "tlog.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16L) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) #define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) @@ -104,7 +104,7 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) { int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { - uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", + uDebug("current capacity:%"PRIzu", maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", pHashObj->capacity, HASH_MAX_CAPACITY); return; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index dac9b85740..b9b365fb42 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -131,7 +131,7 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { COPY_SCALAR_FIELD(placeholderNo); COPY_SCALAR_FIELD(typeData); COPY_SCALAR_FIELD(unit); - if (!pSrc->translate) { + if (!pSrc->translate || pSrc->isNull) { return TSDB_CODE_SUCCESS; } switch (pSrc->node.resType.type) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e401a3da7f..f0f1fc9975 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3011,7 +3011,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkValueIsNull, pNode->isNull); } - if (TSDB_CODE_SUCCESS == code && pNode->translate) { + if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = datumToJson(pNode, pJson); } @@ -3161,7 +3161,7 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkValueIsNull, &pNode->isNull); } - if (TSDB_CODE_SUCCESS == code && pNode->translate) { + if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = jsonToDatum(pJson, pNode); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index cdc4e66e42..9e89955ae5 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -478,7 +478,10 @@ static int32_t tlvDecodeValueEnum(STlvDecoder* pDecoder, void* pValue, int16_t l return code; } -static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) { +static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue, int32_t size) { + if (pTlv->len > size - 1) { + return TSDB_CODE_FAILED; + } memcpy(pValue, pTlv->value, pTlv->len); return TSDB_CODE_SUCCESS; } @@ -919,9 +922,14 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) { } break; } - case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_JSON: { + if (pTlv->len <= 0 || pTlv->len > TSDB_MAX_JSON_TAG_LEN) { + code = TSDB_CODE_FAILED; + break; + } code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p); break; + } case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: // todo @@ -1097,7 +1105,7 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); break; case FUNCTION_CODE_FUNCTION_NAME: - code = tlvDecodeCStr(pTlv, pNode->functionName); + code = tlvDecodeCStr(pTlv, pNode->functionName, sizeof(pNode->functionName)); break; case FUNCTION_CODE_FUNCTION_ID: code = tlvDecodeI32(pTlv, &pNode->funcId); @@ -1226,10 +1234,10 @@ static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI32(pTlv, &pNode->acctId); break; case NAME_CODE_DB_NAME: - code = tlvDecodeCStr(pTlv, pNode->dbname); + code = tlvDecodeCStr(pTlv, pNode->dbname, sizeof(pNode->dbname)); break; case NAME_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tname); + code = tlvDecodeCStr(pTlv, pNode->tname, sizeof(pNode->tname)); break; default: break; @@ -1538,7 +1546,7 @@ static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) { tlvForEach(pDecoder, pTlv, code) { switch (pTlv->type) { case EP_CODE_FQDN: - code = tlvDecodeCStr(pTlv, pNode->fqdn); + code = tlvDecodeCStr(pTlv, pNode->fqdn, sizeof(pNode->fqdn)); break; case EP_CODE_port: code = tlvDecodeU16(pTlv, &pNode->port); @@ -3207,7 +3215,7 @@ static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_QUERY_INSERT_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tableName); + code = tlvDecodeCStr(pTlv, pNode->tableName, sizeof(pNode->tableName)); break; case PHY_QUERY_INSERT_CODE_VG_ID: code = tlvDecodeI32(pTlv, &pNode->vgId); @@ -3284,10 +3292,10 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_DELETER_CODE_TABLE_FNAME: - code = tlvDecodeCStr(pTlv, pNode->tableFName); + code = tlvDecodeCStr(pTlv, pNode->tableFName, sizeof(pNode->tableFName)); break; case PHY_DELETER_CODE_TS_COL_NAME: - code = tlvDecodeCStr(pTlv, pNode->tsColName); + code = tlvDecodeCStr(pTlv, pNode->tsColName, sizeof(pNode->tsColName)); break; case PHY_DELETER_CODE_DELETE_TIME_RANGE: code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 6e964fb53a..bc0c4d42bb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -209,6 +209,10 @@ int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) { } SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + if (NULL == pAllocator) { + nodesError("allocator id %" PRIx64 " weak reference failed", allocatorId); + return -1; + } return pAllocator->self; } @@ -1604,7 +1608,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || - QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type); + QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type || QUERY_NODE_CASE_WHEN == type); } bool nodesIsUnaryOp(const SOperatorNode* pOp) { @@ -1716,9 +1720,10 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = 0; if ('\0' == pCol->tableAlias[0]) { - len = sprintf(name, "%s", pCol->colName); + len = snprintf(name, sizeof(name), "%s", pCol->colName); + } else { + len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } - len = sprintf(name, "%s.%s", pCol->tableAlias, pCol->colName); if (NULL == taosHashGet(pCxt->pColHash, name, len)) { pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); if (TSDB_CODE_SUCCESS == pCxt->errCode) { diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index 787abf287e..fb4b46aa35 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -175,8 +175,6 @@ _end: void taosCleanupKeywordsTable(); -SToken tscReplaceStrToken(char **str, SToken *token, const char *newToken); - SToken taosTokenDup(SToken *pToken, char *buf, int32_t len); #ifdef __cplusplus diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2e6f839368..15b939f14b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -518,7 +518,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa if (NULL != pDbName) { COPY_STRING_FORM_ID_TOKEN(realTable->table.dbName, pDbName); } else { - strcpy(realTable->table.dbName, pCxt->pQueryCxt->db); + snprintf(realTable->table.dbName, sizeof(realTable->table.dbName), "%s", pCxt->pQueryCxt->db); } if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) { COPY_STRING_FORM_ID_TOKEN(realTable->table.tableAlias, pTableAlias); @@ -599,7 +599,7 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { state->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == state->pCol) { nodesDestroyNode((SNode*)state); - CHECK_OUT_OF_MEM(state->pCol); + CHECK_OUT_OF_MEM(NULL); } state->pExpr = pExpr; return (SNode*)state; @@ -613,7 +613,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode interval->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == interval->pCol) { nodesDestroyNode((SNode*)interval); - CHECK_OUT_OF_MEM(interval->pCol); + CHECK_OUT_OF_MEM(NULL); } interval->pInterval = pInterval; interval->pOffset = pOffset; @@ -631,7 +631,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == fill->pWStartTs) { nodesDestroyNode((SNode*)fill); - CHECK_OUT_OF_MEM(fill->pWStartTs); + CHECK_OUT_OF_MEM(NULL); } strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart"); return (SNode*)fill; @@ -1495,10 +1495,10 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno CHECK_OUT_OF_MEM(pStmt); pStmt->indexType = type; pStmt->ignoreExists = ignoreExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->dbName, sizeof(pStmt->dbName), "%s", ((SRealTableNode*)pRealTable)->table.dbName); + snprintf(pStmt->tableName, sizeof(pStmt->tableName), "%s", ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pIndexName); nodesDestroyNode(pRealTable); pStmt->pCols = pCols; @@ -1524,8 +1524,8 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->ignoreNotExists = ignoreNotExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); nodesDestroyNode(pIndexName); return (SNode*)pStmt; } @@ -1818,7 +1818,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(pFunc); - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) { nodesDestroyNode((SNode*)pFunc); CHECK_OUT_OF_MEM(NULL); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index eec20f91e4..5b861ef79a 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -378,7 +378,9 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time } else if (pToken->type == TK_TODAY) { ts = taosGetTimestampToday(timePrec); } else if (pToken->type == TK_NK_INTEGER) { - toInteger(pToken->z, pToken->n, 10, &ts); + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } } else { // parse the RFC-3339/ISO-8601 timestamp format string if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -591,8 +593,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } return func(pMsgBuf, &iv, pSchema->bytes, param); } @@ -894,10 +894,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } - val->i64 = iv; break; } @@ -1173,7 +1170,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* NEXT_TOKEN(pCxt->pSql, sToken); SName sname; - createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); + CHECK_CODE(createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&sname, dbFName); strcpy(pCxt->sTableName, sname.tname); @@ -1605,7 +1602,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (!pCxt->pComCxt->needMultiParse) { continue; } else { - parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); + parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); break; } } @@ -1613,7 +1610,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } - parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); @@ -1650,7 +1647,7 @@ static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) { pCxt->pComCxt->needMultiParse = false; return TSDB_CODE_SUCCESS; } - parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); // merge according to vgId if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index e8c877bed2..997e680c99 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -// clang-format off + #include "parInsertData.h" #include "catalog.h" @@ -25,8 +25,8 @@ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; + TSKEY skey; + void* payloadAddr; int16_t index; } SBlockKeyTuple; @@ -194,8 +194,8 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { SEncoder coder = {0}; - char* pBuf; - int32_t len; + char* pBuf; + int32_t len; int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); @@ -211,19 +211,19 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) } } - pBuf= pBlocks->pData + pBlocks->size; + pBuf = pBlocks->pData + pBlocks->size; tEncoderInit(&coder, pBuf, len); - tEncodeSVCreateTbReq(&coder, pCreateTbReq); + int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); tEncoderClear(&coder); - pBlocks->size += len; pBlocks->createTbReqLen = len; - return TSDB_CODE_SUCCESS; + + return code; } -int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, +int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, + int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); @@ -272,12 +272,12 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); -// if (!pDataBlock->cloned) { - // free the refcount for metermeta - taosMemoryFreeClear(pDataBlock->pTableMeta); + // if (!pDataBlock->cloned) { + // free the refcount for metermeta + taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); -// } + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + // } taosMemoryFreeClear(pDataBlock); } @@ -687,16 +687,16 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p STableDataBlocks** p = taosHashIterate(pHashObj, NULL); STableDataBlocks* pOneTableBlock = *p; SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger *pBlkRowMerger = NULL; + SBlockRowMerger* pBlkRowMerger = NULL; while (pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = - getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); + pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId + int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), + TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, + pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); @@ -708,7 +708,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p // the maximum expanded size in byte when a row-wise data is converted to SDataRow format int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen; + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + + pOneTableBlock->createTbReqLen; if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); @@ -861,7 +862,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); if (pBlock->pTableMeta) { - void *pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); if (NULL == pNewMeta) { taosMemoryFreeClear(*pDst); return TSDB_CODE_OUT_OF_MEMORY; @@ -887,20 +888,18 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI } pBlock->vgId = vgId; - + if (pBlock->pTableMeta) { pBlock->pTableMeta->uid = uid; pBlock->pTableMeta->vgId = vgId; } - + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); return TSDB_CODE_SUCCESS; } -STableMeta *qGetTableMetaInDataBlock(void* pDataBlock) { - return ((STableDataBlocks*)pDataBlock)->pTableMeta; -} +STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } void qFreeStmtDataBlock(void* pDataBlock) { if (pDataBlock == NULL) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 345b9cc5b9..76c01cfd90 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -613,27 +613,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { return 0; } -SToken tscReplaceStrToken(char** str, SToken* token, const char* newToken) { - char* src = *str; - size_t nsize = strlen(newToken); - int32_t size = (int32_t)strlen(*str) - token->n + (int32_t)nsize + 1; - int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); - SToken ntoken; - - *str = taosMemoryCalloc(1, size); - - strncpy(*str, src, bsize); - strcat(*str, newToken); - strcat(*str, token->z + token->n); - - ntoken.n = (uint32_t)nsize; - ntoken.z = *str + bsize; - - taosMemoryFreeClear(src); - - return ntoken; -} - SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { SToken t0 = {0}; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b398b645b3..4739a852c0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -292,9 +292,13 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { } else { do { SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (NULL == pTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (pCxt->currLevel == currTotalLevel) { taosArrayPush(pTables, &pTable); if (hasSameTableAlias(pTables)) { + taosArrayDestroy(pTables); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias); } @@ -308,7 +312,7 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { SFullDatabaseName name = {0}; - strcpy(name.fullDbName, pFullDbName); + snprintf(name.fullDbName, sizeof(name.fullDbName), "%s", pFullDbName); return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName)); } @@ -540,15 +544,17 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray if (TSDB_CODE_SUCCESS == code) { code = collectUseTable(pName, pCxt->pTables); } - if (pParCxt->async) { - code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); - } else { - SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, - .requestId = pParCxt->requestId, - .requestObjRefId = pParCxt->requestRid, - .mgmtEps = pParCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS == code) { + if (pParCxt->async) { + code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); + } else { + SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; - code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + } } if (TSDB_CODE_SUCCESS != code) { parserError("0x%" PRIx64 " getTableIndex error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, @@ -990,9 +996,9 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) return pCxt->errCode; } if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.u; + pVal->datum.i = (int64_t)pVal->datum.u; } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.d; + pVal->datum.i = (int64_t)pVal->datum.d; } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { pVal->datum.i = pVal->datum.b; } @@ -1069,7 +1075,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_BIGINT: { code = toInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.i); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_BIGINT(pVal->datum.i))) { + if (strict && TSDB_CODE_SUCCESS != code) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -1888,6 +1894,7 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW pWhenThen->pWhen = pIsTrue; } if (first) { + first = false; pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; } else if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { SNode* pCastFunc = NULL; @@ -2350,7 +2357,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo } static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSelect) { - if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) { + if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || + TSDB_SYSTEM_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType) { return TSDB_CODE_SUCCESS; } @@ -3429,10 +3437,10 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return NULL; } pCol->node.resType = ((SExprNode*)pNode)->resType; - strcpy(pCol->tableAlias, pTableAlias); - strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); - strcpy(pCol->node.aliasName, pCol->colName); - strcpy(pCol->node.userAlias, ((SExprNode*)pNode)->userAlias); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pTableAlias); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", ((SExprNode*)pNode)->aliasName); + snprintf(pCol->node.aliasName, sizeof(pCol->node.aliasName), "%s", pCol->colName); + snprintf(pCol->node.userAlias, sizeof(pCol->node.userAlias), "%s", ((SExprNode*)pNode)->userAlias); return (SNode*)pCol; } @@ -3739,8 +3747,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int32_t val, int32_t minVal, int32_t maxVal) { if (val >= 0 && (val < minVal || val > maxVal)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 " valid range: [%d, %d]", pName, - val, minVal, maxVal); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %d valid range: [%d, %d]", pName, val, + minVal, maxVal); } return TSDB_CODE_SUCCESS; } @@ -3877,7 +3885,7 @@ static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pO static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) { if (val >= 0 && val != v1 && val != v2) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option %s: %" PRId64 ", only %d, %d allowed", pName, val, v1, v2); + "Invalid option %s: %d, only %d, %d allowed", pName, val, v1, v2); } return TSDB_CODE_SUCCESS; } @@ -4462,8 +4470,8 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pTable->table.dbName, pInfo->pDbName); - strcpy(pTable->table.tableName, pInfo->pTableName); + snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); + snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); pSelect->pFromTable = (SNode*)pTable; @@ -4939,9 +4947,9 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt alterReq.superUser = 0; alterReq.enable = pStmt->enable; alterReq.sysInfo = pStmt->sysinfo; - strcpy(alterReq.pass, pStmt->password); + snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password); if (NULL != pCxt->pParseCxt->db) { - strcpy(alterReq.dbname, pCxt->pParseCxt->db); + snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db); } return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq); @@ -6048,7 +6056,7 @@ static SNode* createProjectCol(const char* pProjCol) { if (NULL == pCol) { return NULL; } - strcpy(pCol->colName, pProjCol); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pProjCol); return (SNode*)pCol; } @@ -6081,9 +6089,9 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32 nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pRealTable->table.dbName, pDb); - strcpy(pRealTable->table.tableName, pTable); - strcpy(pRealTable->table.tableAlias, pTable); + snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); + snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); + snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); pSelect->pFromTable = (SNode*)pRealTable; if (numOfProjs >= 0) { @@ -7124,8 +7132,9 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p pReq->newComment = strdup(pStmt->pOptions->comment); if (NULL == pReq->newComment) { code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pReq->newCommentLen = strlen(pReq->newComment); } - pReq->newCommentLen = strlen(pReq->newComment); } else { pReq->newCommentLen = -1; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dab1f01574..ee82e6c04b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -381,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); + taosMemoryFree(tmp); goto end; } val.nData = valLen; @@ -652,8 +653,8 @@ static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCac } SUserAuthInfo auth = {0}; - strcpy(auth.user, pCxt->pUser); - strcpy(auth.dbFName, p->dbFName); + snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser); + snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName); auth.type = AUTH_TYPE_WRITE; taosArrayPush(pCatalogReq->pUser, &auth); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e6868f0ceb..9e1a01b13d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -66,7 +66,8 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { switch (nodeType(*pNode)) { case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { + case QUERY_NODE_FUNCTION: + case QUERY_NODE_CASE_WHEN: { SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; SNode* pExpr; int32_t index = 0; @@ -118,6 +119,17 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } +static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { + nodesWalkExpr(pExpr, doNameExpr, NULL); + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL}; + cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); + if (TSDB_CODE_SUCCESS == cxt.errCode) { + nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); + nodesClearList(cxt.pExprs); + } + return cxt.errCode; +} + static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; @@ -459,9 +471,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) return NULL; } pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pExpr->aliasName); if (NULL != pStmtName) { - strcpy(pCol->tableAlias, pStmtName); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName); } return pCol; } @@ -711,8 +723,13 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo nodesDestroyNode((SNode*)pWindow); return TSDB_CODE_OUT_OF_MEMORY; } - - return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); + // rewrite the expression in subsequent clauses + int32_t code = rewriteExprForSelect(pWindow->pStateExpr, pSelect, SQL_CLAUSE_WINDOW); + if (TSDB_CODE_SUCCESS == code) { + code = createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); + } + + return code; } static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 50f1f4b369..7fd90dd4e8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -124,9 +124,8 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order) { EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - // *((bool*)pContext) = - // (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); - *((bool*)pContext) = true; + *((bool*)pContext) = + (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -1124,7 +1123,7 @@ static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, SNodeList** pS bool notOptimize = false; int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, ¬Optimize, pSequencingNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { - nodesClearList(*pSequencingNodes); + NODES_CLEAR_LIST(*pSequencingNodes); } return code; } @@ -1361,74 +1360,6 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo return code; } -static SNode* smaIndexOptCreateWStartTs() { - SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pWStart) { - return NULL; - } - strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pWStart, NULL, 0)) { - nodesDestroyNode((SNode*)pWStart); - return NULL; - } - return (SNode*)pWStart; -} - -static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { - SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pMergeKey) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->pExpr = nodesCloneNode(pCol); - if (NULL == pMergeKey->pExpr) { - nodesDestroyNode((SNode*)pMergeKey); - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->order = ORDER_ASC; - pMergeKey->nullOrder = NULL_ORDER_FIRST; - return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); -} - -static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { - if (wstrartIndex < 0) { - SNode* pWStart = smaIndexOptCreateWStartTs(); - if (NULL == pWStart) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t code = createColumnByRewriteExpr(pWStart, &pInterval->node.pTargets); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pWStart); - return code; - } - wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; - } - return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); -} - -static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { - SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; - SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); - if (NULL == pMergeTargets) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SLogicNode* pSmaScan = NULL; - SLogicNode* pMerge = NULL; - SNodeList* pMergeKeys = NULL; - int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); - } - return code; -} - static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pSmaCols, int32_t wstrartIndex) { SLogicNode* pSmaScan = NULL; @@ -1559,7 +1490,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) { return NULL; } - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { SColumnNode* pCol = (SColumnNode*)pNode; partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName); @@ -2028,7 +1959,7 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode if (NULL != pSelectValue) { strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); } else { - snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc); + snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc); } int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol)); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 2935c647aa..8ff8538c0d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -67,7 +67,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const S if (NULL == pSlot) { return NULL; } - strcpy(pSlot->name, pName); + snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -663,13 +663,17 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); } - SNodeList* condCols = nodesMakeList(); if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { - code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols); - } - if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc); - nodesDestroyList(condCols); + SNodeList* pCondCols = nodesMakeList(); + if (NULL == pCondCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &pCondCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pCondCols, pJoin->node.pOutputDataBlockDesc); + } + nodesDestroyList(pCondCols); } if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { @@ -761,7 +765,8 @@ static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: { + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_CASE_WHEN: { return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_FUNCTION: { @@ -1633,7 +1638,7 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; if (NULL != pCxt->pPlanCxt->pUser) { - strcpy(pSubplan->user, pCxt->pPlanCxt->pUser); + snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser); } return pSubplan; } @@ -1824,7 +1829,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNode* pSubplan, int32_t lev return TSDB_CODE_OUT_OF_MEMORY; } } - return nodesListStrictAppend(pGroup->pNodeList, (SNode*)pSubplan); + return nodesListAppend(pGroup->pNodeList, (SNode*)pSubplan); } static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 74ed3b57a4..9567a2388f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -342,7 +342,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart); int32_t code = fmGetFuncInfo(pWStart, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pFuncs, (SNode*)pWStart); @@ -367,7 +367,7 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWEnd->functionName, "_wend"); - snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, pWEnd); + snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd); int32_t code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd); @@ -389,22 +389,19 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic SNode* pConditions = pMergeWindow->node.pConditions; pMergeWindow->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SWindowLogicNode* pPartWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow); if (NULL == pPartWin) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartWin->node.groupAction = GROUP_ACTION_KEEP; + pMergeWindow->node.pTargets = pTargets; + pMergeWindow->node.pConditions = pConditions; + pPartWin->node.pChildren = pChildren; + splSetParent((SLogicNode*)pPartWin); - if (TSDB_CODE_SUCCESS == code) { - pMergeWindow->node.pTargets = pTargets; - pMergeWindow->node.pConditions = pConditions; - pPartWin->node.pChildren = pChildren; - splSetParent((SLogicNode*)pPartWin); - code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); - } int32_t index = 0; + int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = stbSplAppendWStart(pPartWin->pFuncs, &index); } @@ -721,15 +718,16 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO SNode* pConditions = pMergeAgg->node.pConditions; pMergeAgg->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SAggLogicNode* pPartAgg = (SAggLogicNode*)nodesCloneNode((SNode*)pMergeAgg); if (NULL == pPartAgg) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartAgg->node.groupAction = GROUP_ACTION_KEEP; - if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index ac1589bec0..a13e959a36 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -53,7 +53,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { + case QUERY_NODE_FUNCTION: + case QUERY_NODE_CASE_WHEN: { SExprNode* pExpr = (SExprNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 621a38ea4a..80ba356e8f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -24,8 +24,8 @@ #pragma GCC diagnostic ignored "-Wformat-truncation" #endif -int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallocFp)(int32_t)) = {0}; +int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallocFp)(int64_t)) = {0}; int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0}; int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { @@ -67,8 +67,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { return TSDB_CODE_SUCCESS; } -int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallcFp)(int32_t)) { +int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallcFp)(int64_t)) { SBuildTableInput *pInput = input; if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -91,7 +91,7 @@ int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { SBuildUseDBInput *pInput = input; if (NULL == pInput || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -114,7 +114,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms return TSDB_CODE_SUCCESS; } -int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -132,7 +132,7 @@ int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -150,7 +150,7 @@ int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -167,7 +167,7 @@ int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -185,7 +185,7 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -203,8 +203,8 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallcFp)(int32_t)) { +int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -227,7 +227,7 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -245,7 +245,7 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -263,7 +263,7 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 626e78b4d3..36c6817595 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -123,6 +123,7 @@ typedef struct SQWTaskCtx { int32_t execId; int32_t level; + bool queryGotData; bool queryRsped; bool queryEnd; bool queryContinue; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9115c2d3aa..d2c16a1cb5 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -10,6 +10,7 @@ #include "tdatablock.h" #include "tmsg.h" #include "tname.h" +#include "tglobal.h" SQWorkerMgmt gQwMgmt = { .lock = 0, @@ -92,6 +93,19 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { return TSDB_CODE_SUCCESS; } +int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) { + if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) { + if (!ctx->localExec) { + qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode)); + } + + ctx->queryRsped = true; + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; @@ -144,7 +158,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (numOfResBlock == 0) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); } else { - QW_TASK_DLOG("qExecTask done", ""); + QW_TASK_DLOG("qExecTask done, useconds:%" PRIu64, useconds); } dsEndPut(sinkHandle, useconds); @@ -234,6 +248,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t code = 0; SOutputData output = {0}; + if (NULL == ctx->sinkHandle) { + return TSDB_CODE_SUCCESS; + } + *dataLen = 0; while (true) { @@ -407,6 +425,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (!ctx->queryRsped) { QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); @@ -419,6 +442,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); @@ -499,21 +527,17 @@ _return: if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC); + ctx->queryGotData = true; } - if (QW_PHASE_POST_QUERY == phase && ctx) { - if (!ctx->localExec) { - bool rsped = false; - SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; - qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); - qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); - if (!rsped) { - qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); - } + if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) { + bool rsped = false; + SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + if (!rsped) { + qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false); } - - ctx->queryRsped = true; } if (ctx) { @@ -551,6 +575,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); ctx->ctrlConnInfo = qwMsg->connInfo; + ctx->phase = -1; QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); @@ -604,6 +629,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } + qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); @@ -619,6 +646,31 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); + if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + void *rsp = NULL; + int32_t dataLen = 0; + SOutputData sOutput = {0}; + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + + if (rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + } + + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, + tstrerror(code), dataLen); + } + } + QW_RET(TSDB_CODE_SUCCESS); } @@ -740,7 +792,9 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { locked = true; // RC WARNING - if (QW_QUERY_RUNNING(ctx)) { + if (-1 == ctx->phase || false == ctx->queryGotData) { + QW_TASK_DLOG_E("task query unfinished"); + } else if (QW_QUERY_RUNNING(ctx)) { atomic_store_8((int8_t *)&ctx->queryContinue, 1); } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 0a9913e004..d3f29c0e49 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -86,7 +86,7 @@ typedef struct SScalarCtx { } \ } while (0) -int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow); +int32_t sclConvertValueToSclParam(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow); int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); @@ -95,6 +95,11 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) void sclFreeParam(SScalarParam* param); +void doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr); +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr); +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr); #ifdef __cplusplus } diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 1a749c61b3..e633b39223 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -20,6 +20,15 @@ extern "C" { #endif +typedef struct SSclVectorConvCtx { + const SScalarParam* pIn; + SScalarParam* pOut; + int32_t startIndex; + int32_t endIndex; + int16_t inType; + int16_t outType; +} SSclVectorConvCtx; + typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); static FORCE_INLINE double getVectorDoubleValue_TINYINT(void *src, int32_t index) { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 132e0ed214..81ee173648 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1165,7 +1165,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { SValueNode *valueNode = (SValueNode *)cell->pNode; if (valueNode->node.resType.type != type) { int32_t overflow = 0; - code = doConvertDataType(valueNode, &out, &overflow); + code = sclConvertValueToSclParam(valueNode, &out, &overflow); if (code) { // fltError("convert from %d to %d failed", in.type, out.type); FLT_ERR_RET(code); @@ -1973,7 +1973,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } // todo refactor the convert - int32_t code = doConvertDataType(var, &out, NULL); + int32_t code = sclConvertValueToSclParam(var, &out, NULL); if (code != TSDB_CODE_SUCCESS) { qError("convert value to type[%d] failed", type); return TSDB_CODE_TSC_INVALID_OPERATION; @@ -3792,6 +3792,11 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { return DEAL_RES_CONTINUE; } + if (QUERY_NODE_CASE_WHEN == nodeType(*pNode) || QUERY_NODE_WHEN_THEN == nodeType(*pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { SOperatorNode *node = (SOperatorNode *)*pNode; if (!FLT_IS_COMPARISON_OPERATOR(node->opType)) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index da7e286036..1f09fd4799 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -61,7 +61,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -int32_t doConvertDataType(SValueNode *pValueNode, SScalarParam *out, int32_t *overflow) { +int32_t sclConvertValueToSclParam(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow) { SScalarParam in = {.numOfRows = 1}; int32_t code = sclCreateColumnInfoData(&pValueNode->node.resType, 1, &in); if (code != TSDB_CODE_SUCCESS) { @@ -71,12 +71,34 @@ int32_t doConvertDataType(SValueNode *pValueNode, SScalarParam *out, int32_t *ov colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colInfoDataEnsureCapacity(out->columnData, 1); - code = vectorConvertImpl(&in, out, overflow); + code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); sclFreeParam(&in); return code; } +int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { + SSDataBlock* pb = taosArrayGetP(pBlockList, 0); + SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); + if (NULL == pLeft) { + sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pLeft->numOfRows = pb->info.rows; + + if (pDst->numOfRows < pb->info.rows) { + colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows); + } + + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); + OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); + + taosMemoryFree(pLeft); + + return TSDB_CODE_SUCCESS; +} + int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { @@ -110,7 +132,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { } int32_t overflow = 0; - code = doConvertDataType(valueNode, &out, &overflow); + code = sclConvertValueToSclParam(valueNode, &out, &overflow); if (code != TSDB_CODE_SUCCESS) { // sclError("convert data from %d to %d failed", in.type, out.type); SCL_ERR_JRET(code); @@ -178,7 +200,7 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParam(SScalarParam *param) { - if (!param->colAlloced) { + if (NULL == param || !param->colAlloced) { return; } @@ -386,7 +408,8 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t } case QUERY_NODE_FUNCTION: case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: { + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_CASE_WHEN: { SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); @@ -540,6 +563,135 @@ _return: SCL_RET(code); } +int32_t sclGetNodeRes(SNode* node, SScalarCtx *ctx, SScalarParam **res) { + if (NULL == node) { + return TSDB_CODE_SUCCESS; + } + + int32_t rowNum = 0; + *res = taosMemoryCalloc(1, sizeof(**res)); + if (NULL == *res) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCL_ERR_RET(sclInitParam(node, *res, ctx, &rowNum)); + + return TSDB_CODE_SUCCESS; +} + +int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCell, SScalarParam *pCase, SScalarParam *pElse, SScalarParam *pComp, SScalarParam *output, int32_t rowIdx, int32_t totalRows, bool *complete) { + SNode *node = NULL; + SWhenThenNode* pWhenThen = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + int32_t code = 0; + + for (SListCell* cell = pCell; (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) { + pWhenThen = (SWhenThenNode*)node; + + SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL); + + bool *equal = (bool*)colDataGetData(pComp->columnData, rowIdx); + if (*equal) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0))); + + if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + } + + if (pElse) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0))); + + if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pElse->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + colDataAppend(output->columnData, rowIdx, NULL, true); + + if (0 == rowIdx && 1 == pCase->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + +_return: + + sclFreeParam(pWhen); + sclFreeParam(pThen); + + SCL_RET(code); +} + +int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCell, SScalarParam *pElse, SScalarParam *output, + int32_t rowIdx, int32_t totalRows, bool *complete, bool preSingle) { + SNode *node = NULL; + SWhenThenNode* pWhenThen = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + int32_t code = 0; + + for (SListCell* cell = pCell; (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) { + pWhenThen = (SWhenThenNode*)node; + pWhen = NULL; + pThen = NULL; + + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + bool *whenValue = (bool*)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? rowIdx : 0)); + + if (*whenValue) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0))); + + if (preSingle && 0 == rowIdx && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + sclFreeParam(pWhen); + sclFreeParam(pThen); + } + + if (pElse) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0))); + + if (preSingle && 0 == rowIdx && 1 == pElse->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + colDataAppend(output->columnData, rowIdx, NULL, true); + + if (preSingle && 0 == rowIdx && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + +_return: + + sclFreeParam(pWhen); + sclFreeParam(pThen); + + SCL_RET(code); +} + int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { SScalarParam *params = NULL; int32_t rowNum = 0; @@ -698,6 +850,101 @@ _return: SCL_RET(code); } +int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *output) { + int32_t code = 0; + SScalarParam *pCase = NULL; + SScalarParam *pElse = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + SScalarParam comp = {0}; + int32_t rowNum = 1; + bool complete = false; + + if (NULL == node->pWhenThenList || node->pWhenThenList->length <= 0) { + sclError("invalid whenThen list"); + SCL_ERR_RET(TSDB_CODE_INVALID_PARA); + } + + if (ctx->pBlockList) { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, 0); + rowNum = pb->info.rows; + output->numOfRows = pb->info.rows; + } + + SCL_ERR_JRET(sclCreateColumnInfoData(&node->node.resType, rowNum, output)); + + SCL_ERR_JRET(sclGetNodeRes(node->pCase, ctx, &pCase)); + SCL_ERR_JRET(sclGetNodeRes(node->pElse, ctx, &pElse)); + + SDataType compType = {0}; + compType.type = TSDB_DATA_TYPE_BOOL; + compType.bytes = tDataTypes[compType.type].bytes; + + SCL_ERR_JRET(sclCreateColumnInfoData(&compType, rowNum, &comp)); + + SNode* tnode = NULL; + SWhenThenNode* pWhenThen = (SWhenThenNode*)node->pWhenThenList->pHead->pNode; + + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + if (pCase) { + vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL); + + for (int32_t i = 0; i < rowNum; ++i) { + bool *equal = (bool*)colDataGetData(comp.columnData, (comp.numOfRows > 1 ? i : 0)); + if (*equal) { + colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); + if (0 == i && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + break; + } + } else { + SCL_ERR_JRET(sclWalkCaseWhenList(ctx, node->pWhenThenList, node->pWhenThenList->pHead->pNext, pCase, pElse, &comp, output, i, rowNum, &complete)); + if (complete) { + break; + } + } + } + } else { + for (int32_t i = 0; i < rowNum; ++i) { + bool *whenValue = (bool*)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? i : 0)); + if (*whenValue) { + colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); + if (0 == i && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + break; + } + } else { + SCL_ERR_JRET(sclWalkWhenList(ctx, node->pWhenThenList, node->pWhenThenList->pHead->pNext, pElse, output, i, rowNum, &complete, (pWhen->numOfRows == 1 && pThen->numOfRows == 1))); + if (complete) { + break; + } + } + } + } + + sclFreeParam(pCase); + sclFreeParam(pElse); + sclFreeParam(&comp); + sclFreeParam(pWhen); + sclFreeParam(pThen); + + return TSDB_CODE_SUCCESS; + +_return: + + sclFreeParam(pCase); + sclFreeParam(pElse); + sclFreeParam(&comp); + sclFreeParam(pWhen); + sclFreeParam(pThen); + sclFreeParam(output); + + SCL_RET(code); +} + + EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opType) { if (opType <= OP_TYPE_CALC_MAX) { SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); @@ -960,9 +1207,66 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } +EDealRes sclRewriteCaseWhen(SNode** pNode, SScalarCtx *ctx) { + SCaseWhenNode *node = (SCaseWhenNode *)*pNode; + + if ((!SCL_IS_CONST_NODE(node->pCase)) || (!SCL_IS_CONST_NODE(node->pElse))) { + return DEAL_RES_CONTINUE; + } + + SNode* tnode = NULL; + FOREACH(tnode, node->pWhenThenList) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)tnode; + if (!SCL_IS_CONST_NODE(pWhenThen->pWhen) || !SCL_IS_CONST_NODE(pWhenThen->pThen)) { + return DEAL_RES_CONTINUE; + } + } + + SScalarParam output = {0}; + ctx->code = sclExecCaseWhen(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->translate = true; + + res->node.resType = node->node.resType; + if (colDataIsNull_s(output.columnData, 0)) { + res->isNull = true; + res->node.resType = node->node.resType; + } else { + int32_t type = output.columnData->info.type; + if (IS_VAR_DATA_TYPE(type)) { // todo refactor + res->datum.p = output.columnData->pData; + output.columnData->pData = NULL; + } else { + nodesSetValueNodeValue(res, output.columnData->pData); + } + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + return DEAL_RES_CONTINUE; +} + + EDealRes sclConstantsRewriter(SNode **pNode, void *pContext) { SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + return sclRewriteOperator(pNode, ctx); + } + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { return sclRewriteFunction(pNode, ctx); } @@ -971,8 +1275,8 @@ EDealRes sclConstantsRewriter(SNode **pNode, void *pContext) { return sclRewriteLogic(pNode, ctx); } - if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { - return sclRewriteOperator(pNode, ctx); + if (QUERY_NODE_CASE_WHEN == nodeType(*pNode)) { + return sclRewriteCaseWhen(pNode, ctx); } return DEAL_RES_CONTINUE; @@ -1082,13 +1386,36 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } +EDealRes sclWalkCaseWhen(SNode* pNode, SScalarCtx *ctx) { + SCaseWhenNode *node = (SCaseWhenNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecCaseWhen(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + EDealRes sclCalcWalker(SNode *pNode, void *pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || - QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_LEFT_VALUE == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) + || QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_LEFT_VALUE == nodeType(pNode) + || QUERY_NODE_WHEN_THEN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + return sclWalkOperator(pNode, ctx); + } + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { return sclWalkFunction(pNode, ctx); } @@ -1097,38 +1424,19 @@ EDealRes sclCalcWalker(SNode *pNode, void *pContext) { return sclWalkLogic(pNode, ctx); } - if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - return sclWalkOperator(pNode, ctx); - } - if (QUERY_NODE_TARGET == nodeType(pNode)) { return sclWalkTarget(pNode, ctx); } + if (QUERY_NODE_CASE_WHEN == nodeType(pNode)) { + return sclWalkCaseWhen(pNode, ctx); + } + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } -int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { - SSDataBlock *pb = taosArrayGetP(pBlockList, 0); - SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); - if (NULL == pLeft) { - sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pLeft->numOfRows = pb->info.rows; - colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows); - - _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); - OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); - - taosMemoryFree(pLeft); - - return TSDB_CODE_SUCCESS; -} - int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) { if (NULL == pNode) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e83fe7ecd0..f467721248 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -367,46 +367,45 @@ static FORCE_INLINE void ncharToVar(char *buf, SScalarParam *pOut, int32_t rowIn taosMemoryFree(t); } -// TODO opt performance, tmp is not needed. -int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, int32_t inType, int32_t outType, - int32_t *overflow) { +//TODO opt performance, tmp is not needed. +int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t* overflow) { bool vton = false; _bufConverteFunc func = NULL; - if (TSDB_DATA_TYPE_BOOL == outType) { + if (TSDB_DATA_TYPE_BOOL == pCtx->outType) { func = varToBool; - } else if (IS_SIGNED_NUMERIC_TYPE(outType)) { + } else if (IS_SIGNED_NUMERIC_TYPE(pCtx->outType)) { func = varToSigned; - } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->outType)) { func = varToUnsigned; - } else if (IS_FLOAT_TYPE(outType)) { + } else if (IS_FLOAT_TYPE(pCtx->outType)) { func = varToFloat; - } else if (outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary - ASSERT(inType == TSDB_DATA_TYPE_NCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary + ASSERT(pCtx->inType == TSDB_DATA_TYPE_NCHAR); func = ncharToVar; vton = true; - } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar - ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar + ASSERT(pCtx->inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; vton = true; - } else if (TSDB_DATA_TYPE_TIMESTAMP == outType) { + } else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) { func = varToTimestamp; } else { - sclError("invalid convert outType:%d", outType); + sclError("invalid convert outType:%d", pCtx->outType); return TSDB_CODE_QRY_APP_ERROR; } - pOut->numOfRows = pIn->numOfRows; - for (int32_t i = 0; i < pIn->numOfRows; ++i) { - if (IS_HELPER_NULL(pIn->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); + pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { + if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) { + colDataAppendNULL(pCtx->pOut->columnData, i); continue; } - char *data = colDataGetVarData(pIn->columnData, i); - int32_t convertType = inType; - if (inType == TSDB_DATA_TYPE_JSON) { - if (*data == TSDB_DATA_TYPE_NULL) { + char* data = colDataGetVarData(pCtx->pIn->columnData, i); + int32_t convertType = pCtx->inType; + if(pCtx->inType == TSDB_DATA_TYPE_JSON){ + if(*data == TSDB_DATA_TYPE_NULL) { ASSERT(0); } else if (*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; @@ -415,13 +414,13 @@ int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, in terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; return terrno; } else { - convertNumberToNumber(data + CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType); + convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pCtx->pOut->columnData, i), *data, pCtx->outType); continue; } } - int32_t bufSize = pIn->columnData->info.bytes; - char *tmp = taosMemoryMalloc(varDataTLen(data)); - if (!tmp) { + int32_t bufSize = pCtx->pIn->columnData->info.bytes; + char *tmp = taosMemoryMalloc(varDataTLen(data)); + if(!tmp){ sclError("out of memory in vectorConvertFromVarData"); return TSDB_CODE_OUT_OF_MEMORY; } @@ -444,8 +443,8 @@ int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, in tmp[len] = 0; } } - - (*func)(tmp, pOut, i, overflow); + + (*func)(tmp, pCtx->pOut, i, overflow); taosMemoryFreeClear(tmp); } @@ -581,64 +580,64 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t return true; } -int32_t vectorConvertToVarData(const SScalarParam *pIn, SScalarParam *pOut, int16_t inType, int16_t outType) { - SColumnInfoData *pInputCol = pIn->columnData; - SColumnInfoData *pOutputCol = pOut->columnData; - char tmp[128] = {0}; +int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { + SColumnInfoData* pInputCol = pCtx->pIn->columnData; + SColumnInfoData* pOutputCol = pCtx->pOut->columnData; + char tmp[128] = {0}; - if (IS_SIGNED_NUMERIC_TYPE(inType) || inType == TSDB_DATA_TYPE_BOOL || inType == TSDB_DATA_TYPE_TIMESTAMP) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inType) || pCtx->inType == TSDB_DATA_TYPE_BOOL || pCtx->inType == TSDB_DATA_TYPE_TIMESTAMP) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } - } else if (IS_UNSIGNED_NUMERIC_TYPE(inType)) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inType)) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint64_t, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } - } else if (IS_FLOAT_TYPE(inType)) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + } else if (IS_FLOAT_TYPE(pCtx->inType)) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, double, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%lf", value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } } else { - sclError("not supported input type:%d", inType); + sclError("not supported input type:%d", pCtx->inType); return TSDB_CODE_QRY_APP_ERROR; } @@ -646,34 +645,35 @@ int32_t vectorConvertToVarData(const SScalarParam *pIn, SScalarParam *pOut, int1 } // TODO opt performance -int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow) { - SColumnInfoData *pInputCol = pIn->columnData; - SColumnInfoData *pOutputCol = pOut->columnData; +int32_t vectorConvertSingleColImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* overflow, int32_t startIndex, int32_t numOfRows) { + SColumnInfoData* pInputCol = pIn->columnData; + SColumnInfoData* pOutputCol = pOut->columnData; if (NULL == pInputCol) { sclError("input column is NULL, hashFilter %p", pIn->pHashFilter); return TSDB_CODE_APP_ERROR; } - int16_t inType = pInputCol->info.type; - int16_t outType = pOutputCol->info.type; + int32_t rstart = startIndex >= 0 ? startIndex : 0; + int32_t rend = numOfRows > 0 ? rstart + numOfRows - 1 : rstart + pIn->numOfRows - 1; + SSclVectorConvCtx cCtx = {pIn, pOut, rstart, rend, pInputCol->info.type, pOutputCol->info.type}; - if (IS_VAR_DATA_TYPE(inType)) { - return vectorConvertFromVarData(pIn, pOut, inType, outType, overflow); + if (IS_VAR_DATA_TYPE(cCtx.inType)) { + return vectorConvertFromVarData(&cCtx, overflow); } if (overflow) { ASSERT(1 == pIn->numOfRows); pOut->numOfRows = 0; - - if (IS_SIGNED_NUMERIC_TYPE(outType)) { - int64_t minValue = tDataTypes[outType].minValue; - int64_t maxValue = tDataTypes[outType].maxValue; - + + if (IS_SIGNED_NUMERIC_TYPE(cCtx.outType)) { + int64_t minValue = tDataTypes[cCtx.outType].minValue; + int64_t maxValue = tDataTypes[cCtx.outType].maxValue; + double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -683,13 +683,13 @@ int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t * } else { *overflow = 0; } - } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { - uint64_t minValue = (uint64_t)tDataTypes[outType].minValue; - uint64_t maxValue = (uint64_t)tDataTypes[outType].maxValue; - + } else if (IS_UNSIGNED_NUMERIC_TYPE(cCtx.outType)) { + uint64_t minValue = (uint64_t)tDataTypes[cCtx.outType].minValue; + uint64_t maxValue = (uint64_t)tDataTypes[cCtx.outType].maxValue; + double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -703,157 +703,157 @@ int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t * } pOut->numOfRows = pIn->numOfRows; - switch (outType) { + switch (cCtx.outType) { case TSDB_DATA_TYPE_BOOL: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } bool value = 0; - GET_TYPED_DATA(value, bool, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, bool, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } case TSDB_DATA_TYPE_TINYINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int8_t value = 0; - GET_TYPED_DATA(value, int8_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int8_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } - case TSDB_DATA_TYPE_SMALLINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_SMALLINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int16_t value = 0; - GET_TYPED_DATA(value, int16_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int16_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt16(pOutputCol, i, (int16_t *)&value); } break; } - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_INT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int32_t value = 0; - GET_TYPED_DATA(value, int32_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int32_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt32(pOutputCol, i, (int32_t *)&value); } break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt64(pOutputCol, i, (int64_t *)&value); } break; } - case TSDB_DATA_TYPE_UTINYINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_UTINYINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint8_t value = 0; - GET_TYPED_DATA(value, uint8_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint8_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } - case TSDB_DATA_TYPE_USMALLINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_USMALLINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint16_t value = 0; - GET_TYPED_DATA(value, uint16_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint16_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt16(pOutputCol, i, (int16_t *)&value); } break; } - case TSDB_DATA_TYPE_UINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_UINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint32_t value = 0; - GET_TYPED_DATA(value, uint32_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint32_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt32(pOutputCol, i, (int32_t *)&value); } break; } case TSDB_DATA_TYPE_UBIGINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); - colDataAppendInt64(pOutputCol, i, (int64_t *)&value); + GET_TYPED_DATA(value, uint64_t, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendInt64(pOutputCol, i, (int64_t*)&value); } break; } - case TSDB_DATA_TYPE_FLOAT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_FLOAT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } float value = 0; - GET_TYPED_DATA(value, float, inType, colDataGetData(pInputCol, i)); - colDataAppendFloat(pOutputCol, i, (float *)&value); + GET_TYPED_DATA(value, float, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendFloat(pOutputCol, i, (float*)&value); } break; } case TSDB_DATA_TYPE_DOUBLE: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); - colDataAppendDouble(pOutputCol, i, (double *)&value); + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendDouble(pOutputCol, i, (double*)&value); } break; } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { - return vectorConvertToVarData(pIn, pOut, inType, outType); + return vectorConvertToVarData(&cCtx); } default: - sclError("invalid convert output type:%d", outType); + sclError("invalid convert output type:%d", cCtx.outType); return TSDB_CODE_QRY_APP_ERROR; } @@ -894,7 +894,7 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { return gConvertTypes[type2][type1]; } -int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int32_t type) { +int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows) { SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; @@ -903,17 +903,16 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 return TSDB_CODE_OUT_OF_MEMORY; } - code = vectorConvertImpl(input, output, NULL); + code = vectorConvertSingleColImpl(input, output, NULL, startIndex, numOfRows); if (code) { - // taosMemoryFreeClear(paramOut1->data); return code; } return TSDB_CODE_SUCCESS; } -int32_t vectorConvert(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pLeftOut, SScalarParam *pRightOut) { - int32_t leftType = GET_PARAM_TYPE(pLeft); +int32_t vectorConvertCols(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut, int32_t startIndex, int32_t numOfRows) { + int32_t leftType = GET_PARAM_TYPE(pLeft); int32_t rightType = GET_PARAM_TYPE(pRight); if (leftType == rightType) { return TSDB_CODE_SUCCESS; @@ -941,14 +940,14 @@ int32_t vectorConvert(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p } if (type != GET_PARAM_TYPE(param1)) { - code = vectorConvertScalarParam(param1, paramOut1, type); + code = vectorConvertSingleCol(param1, paramOut1, type, startIndex, numOfRows); if (code) { return code; } } if (type != GET_PARAM_TYPE(param2)) { - code = vectorConvertScalarParam(param2, paramOut2, type); + code = vectorConvertSingleCol(param2, paramOut2, type, startIndex, numOfRows); if (code) { return code; } @@ -962,31 +961,6 @@ enum { VECTOR_UN_CONVERT = 0x2, }; -static int32_t doConvertHelper(SScalarParam *pDest, int32_t *convert, const SScalarParam *pParam, int32_t type) { - SColumnInfoData *pCol = pParam->columnData; - - if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { - pDest->numOfRows = pParam->numOfRows; - - SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - int32_t code = sclCreateColumnInfoData(&t, pParam->numOfRows, pDest); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = vectorConvertImpl(pParam, pDest, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - *convert = VECTOR_DO_CONVERT; - } else { - *convert = VECTOR_UN_CONVERT; - } - - return TSDB_CODE_SUCCESS; -} - // TODO not correct for descending order scan static void vectorMathAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { @@ -1030,20 +1004,25 @@ static void vectorMathTsAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pR } } -static SColumnInfoData *doVectorConvert(SScalarParam *pInput, int32_t *doConvert) { - SScalarParam convertParam = {0}; +static SColumnInfoData* vectorConvertVarToDouble(SScalarParam* pInput, int32_t* converted) { + SScalarParam output = {0}; + SColumnInfoData* pCol = pInput->columnData; - int32_t code = doConvertHelper(&convertParam, doConvert, pInput, TSDB_DATA_TYPE_DOUBLE); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; + if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { + int32_t code = vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, -1, -1); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + *converted = VECTOR_DO_CONVERT; + + return output.columnData; } - if (*doConvert == VECTOR_DO_CONVERT) { - return convertParam.columnData; - } else { - return pInput->columnData; - } + *converted = VECTOR_UN_CONVERT; + + return pInput->columnData; } static void doReleaseVec(SColumnInfoData *pCol, int32_t type) { @@ -1061,9 +1040,9 @@ void vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) || (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) || @@ -1168,9 +1147,9 @@ void vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BIGINT) || (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && @@ -1246,9 +1225,9 @@ void vectorMathMultiply(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1279,9 +1258,9 @@ void vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1333,9 +1312,9 @@ void vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1412,8 +1391,8 @@ void vectorMathMinus(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : (pLeft->numOfRows - 1); int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); + int32_t leftConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); @@ -1525,9 +1504,9 @@ void vectorBitAnd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); @@ -1579,9 +1558,9 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); @@ -1605,8 +1584,8 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, doReleaseVec(pRightCol, rightConvert); } -int32_t doVectorCompareImpl(int32_t numOfRows, SScalarParam *pOut, int32_t startIndex, int32_t step, __compar_fn_t fp, - SScalarParam *pLeft, SScalarParam *pRight, int32_t optr) { +int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t step, __compar_fn_t fp, int32_t optr) { int32_t num = 0; for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { @@ -1659,20 +1638,29 @@ int32_t doVectorCompareImpl(int32_t numOfRows, SScalarParam *pOut, int32_t start return num; } -void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; +void doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr) { + int32_t i = 0; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t lType = GET_PARAM_TYPE(pLeft); int32_t rType = GET_PARAM_TYPE(pRight); __compar_fn_t fp = NULL; - + int32_t compRows = 0; + if (lType == rType) { fp = filterGetCompFunc(lType, optr); } else { fp = filterGetCompFuncEx(lType, rType, optr); } - pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + if (startIndex < 0) { + i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + compRows = pOut->numOfRows; + } else { + compRows = startIndex + numOfRows; + i = startIndex; + } if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { @@ -1690,13 +1678,14 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam * } } } else { // normal compare - pOut->numOfQualified = doVectorCompareImpl(pOut->numOfRows, pOut, i, step, fp, pLeft, pRight, optr); + pOut->numOfQualified = doVectorCompareImpl(pLeft, pRight, pOut, i, compRows, step, fp, optr); } } -void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { - SScalarParam pLeftOut = {0}; - SScalarParam pRightOut = {0}; +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr) { + SScalarParam pLeftOut = {0}; + SScalarParam pRightOut = {0}; SScalarParam *param1 = NULL; SScalarParam *param2 = NULL; @@ -1704,7 +1693,7 @@ void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut param1 = pLeft; param2 = pRight; } else { - vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut); + vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows); if (pLeftOut.columnData != NULL) { param1 = &pLeftOut; @@ -1719,12 +1708,17 @@ void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut } } - vectorCompareImpl(param1, param2, pOut, _ord, optr); + doVectorCompare(param1, param2, pOut, startIndex, numOfRows, _ord, optr); + sclFreeParam(&pLeftOut); sclFreeParam(&pRightOut); } -void vectorGreater(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { + vectorCompareImpl(pLeft, pRight, pOut, -1, -1, _ord, optr); +} + +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_THAN); } @@ -1788,10 +1782,10 @@ void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut pOut->numOfRows = pLeft->numOfRows; } -void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { - vectorConvertImpl(pLeft, pOut, NULL); - for (int32_t i = 0; i < pOut->numOfRows; ++i) { - if (colDataIsNull_s(pOut->columnData, i)) { +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorConvertSingleColImpl(pLeft, pOut, NULL, -1, -1); + for(int32_t i = 0; i < pOut->numOfRows; ++i) { + if(colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; colDataAppendInt8(pOut->columnData, i, &v); colDataSetNotNull_f(pOut->columnData->nullbitmap, i); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7ced4f626c..1a88992840 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -20,13 +20,13 @@ extern "C" { #endif +#include "command.h" #include "os.h" -#include "tarray.h" #include "planner.h" #include "scheduler.h" +#include "tarray.h" #include "thash.h" #include "trpc.h" -#include "command.h" enum { SCH_READ = 1, @@ -51,20 +51,20 @@ typedef enum { SCH_ALL, } SCH_POLICY; -#define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 -#define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 +#define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 +#define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 #define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT -#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ -#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 +#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ +#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 -#define SCH_MAX_TASK_TIMEOUT_USEC 60000000 -#define SCH_DEFAULT_MAX_RETRY_NUM 6 -#define SCH_MIN_AYSNC_EXEC_NUM 3 +#define SCH_MAX_TASK_TIMEOUT_USEC 60000000 +#define SCH_DEFAULT_MAX_RETRY_NUM 6 +#define SCH_MIN_AYSNC_EXEC_NUM 3 typedef struct SSchDebug { - bool lockEnable; - bool apiEnable; + bool lockEnable; + bool apiEnable; } SSchDebug; typedef struct SSchTrans { @@ -80,7 +80,6 @@ typedef struct SSchHbTrans { } SSchHbTrans; typedef struct SSchApiStat { - #if defined(WINDOWS) || defined(_TD_DARWIN_64) size_t avoidCompilationErrors; #endif @@ -88,7 +87,6 @@ typedef struct SSchApiStat { } SSchApiStat; typedef struct SSchRuntimeStat { - #if defined(WINDOWS) || defined(_TD_DARWIN_64) size_t avoidCompilationErrors; #endif @@ -96,7 +94,6 @@ typedef struct SSchRuntimeStat { } SSchRuntimeStat; typedef struct SSchJobStat { - #if defined(WINDOWS) || defined(_TD_DARWIN_64) size_t avoidCompilationErrors; #endif @@ -104,17 +101,17 @@ typedef struct SSchJobStat { } SSchJobStat; typedef struct SSchStat { - SSchApiStat api; - SSchRuntimeStat runtime; - SSchJobStat job; + SSchApiStat api; + SSchRuntimeStat runtime; + SSchJobStat job; } SSchStat; typedef struct SSchResInfo { - SExecResult* execRes; - void** fetchRes; - schedulerExecFp execFp; - schedulerFetchFp fetchFp; - void* cbParam; + SExecResult *execRes; + void **fetchRes; + schedulerExecFp execFp; + schedulerFetchFp fetchFp; + void *cbParam; } SSchResInfo; typedef struct SSchOpEvent { @@ -123,9 +120,9 @@ typedef struct SSchOpEvent { SSchedulerReq *pReq; } SSchOpEvent; -typedef int32_t (*schStatusEnterFp)(void* pHandle, void* pParam); -typedef int32_t (*schStatusLeaveFp)(void* pHandle, void* pParam); -typedef int32_t (*schStatusEventFp)(void* pHandle, void* pParam, void* pEvent); +typedef int32_t (*schStatusEnterFp)(void *pHandle, void *pParam); +typedef int32_t (*schStatusLeaveFp)(void *pHandle, void *pParam); +typedef int32_t (*schStatusEventFp)(void *pHandle, void *pParam, void *pEvent); typedef struct SSchStatusFps { EJobTaskType status; @@ -142,16 +139,16 @@ typedef struct SSchedulerCfg { } SSchedulerCfg; typedef struct SSchedulerMgmt { - uint64_t taskId; // sequential taksId - uint64_t sId; // schedulerId - SSchedulerCfg cfg; - bool exit; - int32_t jobRef; - int32_t jobNum; - SSchStat stat; - SRWLatch hbLock; - SHashObj *hbConnections; - void *queryMgmt; + uint64_t taskId; // sequential taksId + uint64_t sId; // schedulerId + SSchedulerCfg cfg; + bool exit; + int32_t jobRef; + int32_t jobNum; + SSchStat stat; + SRWLatch hbLock; + SHashObj *hbConnections; + void *queryMgmt; } SSchedulerMgmt; typedef struct SSchCallbackParamHeader { @@ -174,11 +171,11 @@ typedef struct SSchHbCallbackParam { } SSchHbCallbackParam; typedef struct SSchFlowControl { - SRWLatch lock; - bool sorted; - int32_t tableNumSum; - uint32_t execTaskNum; - SArray *taskList; // Element is SSchTask* + SRWLatch lock; + bool sorted; + int32_t tableNumSum; + uint32_t execTaskNum; + SArray *taskList; // Element is SSchTask* } SSchFlowControl; typedef struct SSchNodeInfo { @@ -187,50 +184,50 @@ typedef struct SSchNodeInfo { } SSchNodeInfo; typedef struct SSchLevel { - int32_t level; - int8_t status; - SRWLatch lock; - int32_t taskFailed; - int32_t taskSucceed; - int32_t taskNum; - int32_t taskLaunchedNum; - int32_t taskDoneNum; - SArray *subTasks; // Element is SSchTask + int32_t level; + int8_t status; + SRWLatch lock; + int32_t taskFailed; + int32_t taskSucceed; + int32_t taskNum; + int32_t taskLaunchedNum; + int32_t taskDoneNum; + SArray *subTasks; // Element is SSchTask } SSchLevel; typedef struct SSchTaskProfile { - int64_t startTs; - SArray* execTime; - int64_t waitTime; - int64_t endTs; + int64_t startTs; + SArray *execTime; + int64_t waitTime; + int64_t endTs; } SSchTaskProfile; typedef struct SSchTask { - uint64_t taskId; // task id - SRWLatch lock; // task reentrant lock - int32_t maxExecTimes; // task max exec times - int32_t maxRetryTimes; // task max retry times - int32_t retryTimes; // task retry times - bool waitRetry; // wait for retry - int32_t execId; // task current execute index - SSchLevel *level; // level - SRWLatch planLock; // task update plan lock - SSubplan *plan; // subplan - char *msg; // operator tree - int32_t msgLen; // msg length - int8_t status; // task status - int32_t lastMsgType; // last sent msg type - int64_t timeoutUsec; // task timeout useconds before reschedule - SQueryNodeAddr succeedAddr; // task executed success node address - int8_t candidateIdx; // current try condidation index - SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr - SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo - SSchTaskProfile profile; // task execution profile - int32_t childReady; // child task ready number - SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* - SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* - void* handle; // task send handle - bool registerdHb; // registered in hb + uint64_t taskId; // task id + SRWLatch lock; // task reentrant lock + int32_t maxExecTimes; // task max exec times + int32_t maxRetryTimes; // task max retry times + int32_t retryTimes; // task retry times + bool waitRetry; // wait for retry + int32_t execId; // task current execute index + SSchLevel *level; // level + SRWLatch planLock; // task update plan lock + SSubplan *plan; // subplan + char *msg; // operator tree + int32_t msgLen; // msg length + int8_t status; // task status + int32_t lastMsgType; // last sent msg type + int64_t timeoutUsec; // task timeout useconds before reschedule + SQueryNodeAddr succeedAddr; // task executed success node address + int8_t candidateIdx; // current try condidation index + SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr + SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo + SSchTaskProfile profile; // task execution profile + int32_t childReady; // child task ready number + SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* + SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* + void *handle; // task send handle + bool registerdHb; // registered in hb } SSchTask; typedef struct SSchJobAttr { @@ -243,46 +240,46 @@ typedef struct SSchJobAttr { } SSchJobAttr; typedef struct { - int32_t op; - SRWLatch lock; - bool syncReq; + int32_t op; + SRWLatch lock; + bool syncReq; } SSchOpStatus; typedef struct SSchJob { - int64_t refId; - uint64_t queryId; - SSchJobAttr attr; - int32_t levelNum; - int32_t taskNum; - SRequestConnInfo conn; - SArray *nodeList; // qnode/vnode list, SArray - SArray *levels; // starting from 0. SArray - SQueryPlan *pDag; - int64_t allocatorRefId; + int64_t refId; + uint64_t queryId; + SSchJobAttr attr; + int32_t levelNum; + int32_t taskNum; + SRequestConnInfo conn; + SArray *nodeList; // qnode/vnode list, SArray + SArray *levels; // starting from 0. SArray + SQueryPlan *pDag; + int64_t allocatorRefId; - SArray *dataSrcTasks; // SArray - int32_t levelIdx; - SEpSet dataSrcEps; - SHashObj *taskList; - SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* - SHashObj *flowCtrl; // key is ep, element is SSchFlowControl - - SExplainCtx *explainCtx; - int8_t status; - SQueryNodeAddr resNode; - tsem_t rspSem; - SSchOpStatus opStatus; - schedulerChkKillFp chkKillFp; - void* chkKillParam; - SSchTask *fetchTask; - int32_t errCode; - SRWLatch resLock; - SExecResult execRes; - void *fetchRes; //TODO free it or not - bool fetched; - int32_t resNumOfRows; - SSchResInfo userRes; - char *sql; + SArray *dataSrcTasks; // SArray + int32_t levelIdx; + SEpSet dataSrcEps; + SHashObj *taskList; + SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* + SHashObj *flowCtrl; // key is ep, element is SSchFlowControl + + SExplainCtx *explainCtx; + int8_t status; + SQueryNodeAddr resNode; + tsem_t rspSem; + SSchOpStatus opStatus; + schedulerChkKillFp chkKillFp; + void *chkKillParam; + SSchTask *fetchTask; + int32_t errCode; + SRWLatch resLock; + SExecResult execRes; + void *fetchRes; // TODO free it or not + bool fetched; + int32_t resNumOfRows; + SSchResInfo userRes; + char *sql; SQueryProfileSummary summary; } SSchJob; @@ -294,226 +291,282 @@ typedef struct SSchTaskCtx { extern SSchedulerMgmt schMgmt; -#define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) +#define SCH_TASK_TIMEOUT(_task) \ + ((taosGetTimestampUs() - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) #define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) -#define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock) +#define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock) #define SCH_UNLOCK_TASK(_task) SCH_UNLOCK(SCH_WRITE, &(_task)->lock) -#define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) +#define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1) #define SCH_IS_DATA_BIND_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) -#define SCH_IS_DATA_BIND_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) +#define SCH_IS_DATA_BIND_TASK(task) \ + (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) #define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum) -#define SCH_IS_DATA_MERGE_TASK(task) (!SCH_IS_DATA_BIND_TASK(task)) -#define SCH_IS_LOCAL_EXEC_TASK(_job, _task) ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && (!SCH_IS_DATA_BIND_QRY_TASK(_task))) +#define SCH_IS_DATA_MERGE_TASK(task) (!SCH_IS_DATA_BIND_TASK(task)) +#define SCH_IS_LOCAL_EXEC_TASK(_job, _task) \ + ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ + (!SCH_IS_DATA_BIND_QRY_TASK(_task))) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) -#define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) +#define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task)) -#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) +#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) #define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle)) #define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) -#define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) +#define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) #define SCH_GET_JOB_STATUS_STR(job) jobTaskStatusStr(SCH_GET_JOB_STATUS(job)) #define SCH_JOB_IN_SYNC_OP(job) ((job)->opStatus.op && (job)->opStatus.syncReq) -#define SCH_JOB_IN_ASYNC_EXEC_OP(job) ((SCH_OP_EXEC == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_EXEC, SCH_OP_NULL)) && (!(job)->opStatus.syncReq)) -#define SCH_JOB_IN_ASYNC_FETCH_OP(job) ((SCH_OP_FETCH == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_FETCH, SCH_OP_NULL)) && (!(job)->opStatus.syncReq)) +#define SCH_JOB_IN_ASYNC_EXEC_OP(job) \ + ((SCH_OP_EXEC == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_EXEC, SCH_OP_NULL)) && \ + (!(job)->opStatus.syncReq)) +#define SCH_JOB_IN_ASYNC_FETCH_OP(job) \ + ((SCH_OP_FETCH == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_FETCH, SCH_OP_NULL)) && \ + (!(job)->opStatus.syncReq)) #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true -#define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) -#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) -#define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) +#define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) +#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) \ + (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) +#define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY) -#define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } else { (_job)->attr.insertJob = true; } } while (0) -#define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) -#define SCH_IS_INSERT_JOB(_job) ((_job)->attr.insertJob) +#define SCH_SET_JOB_TYPE(_job, type) \ + do { \ + if ((type) != SUBPLAN_TYPE_MODIFY) { \ + (_job)->attr.queryJob = true; \ + } else { \ + (_job)->attr.insertJob = true; \ + } \ + } while (0) +#define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) +#define SCH_IS_INSERT_JOB(_job) ((_job)->attr.insertJob) #define SCH_JOB_NEED_FETCH(_job) ((_job)->attr.needFetch) -#define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job)) -#define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) +#define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job)) +#define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) -#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) -#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)))) -#define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) -#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) (SCH_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) -#define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) +#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) +#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \ + (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)))) +#define SCH_REDIRECT_MSGTYPE(_msgType) \ + ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || \ + (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) +#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) \ + (SCH_REDIRECT_MSGTYPE(_msgType) && \ + (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) +#define SCH_NEED_RETRY(_msgType, _code) \ + ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) -#define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) -#define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps) -#define SCH_TASK_NUM_OF_EPS(_addr) ((_addr)->epSet.numOfEps) +#define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) +#define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps) +#define SCH_TASK_NUM_OF_EPS(_addr) ((_addr)->epSet.numOfEps) -#define SCH_LOG_TASK_START_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ - taosArrayPush((_task)->profile.execTime, &us); \ - if (0 == (_task)->execId) { \ - (_task)->profile.startTs = us; \ - } \ - } while (0) +#define SCH_LOG_TASK_START_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + taosArrayPush((_task)->profile.execTime, &us); \ + if (0 == (_task)->execId) { \ + (_task)->profile.startTs = us; \ + } \ + } while (0) -#define SCH_LOG_TASK_WAIT_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ - (_task)->profile.waitTime += us - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - } while (0) +#define SCH_LOG_TASK_WAIT_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + (_task)->profile.waitTime += us - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + } while (0) - -#define SCH_LOG_TASK_END_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ - int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ +#define SCH_LOG_TASK_END_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - *startts = us - *startts; \ - (_task)->profile.endTs = us; \ - } while (0) - + *startts = us - *startts; \ + (_task)->profile.endTs = us; \ + } while (0) #define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_TLOG(param, ...) \ - qTrace("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_DLOGL(param, ...) \ - qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) \ + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) \ + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_TLOG(param, ...) \ + qTrace("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_DLOGL(param, ...) \ + qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) \ + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) -#define SCH_SET_ERRNO(_err) do { if (TSDB_CODE_SCH_IGNORE_ERROR != (_err)) { terrno = (_err); } } while (0) -#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(_code); return _code; } } while (0) -#define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(_code); } return _code; } while (0) -#define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(code); goto _return; } } while (0) +#define SCH_SET_ERRNO(_err) \ + do { \ + if (TSDB_CODE_SCH_IGNORE_ERROR != (_err)) { \ + terrno = (_err); \ + } \ + } while (0) +#define SCH_ERR_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + SCH_SET_ERRNO(_code); \ + return _code; \ + } \ + } while (0) +#define SCH_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + SCH_SET_ERRNO(_code); \ + } \ + return _code; \ + } while (0) +#define SCH_ERR_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + SCH_SET_ERRNO(code); \ + goto _return; \ + } \ + } while (0) -#define SCH_LOCK_DEBUG(...) do { if (gSCHDebug.lockEnable) { qDebug(__VA_ARGS__); } } while (0) +#define SCH_LOCK_DEBUG(...) \ + do { \ + if (gSCHDebug.lockEnable) { \ + qDebug(__VA_ARGS__); \ + } \ + } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define SCH_LOCK(type, _lock) do { \ - if (SCH_READ == (type)) { \ - assert(atomic_load_32(_lock) >= 0); \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) > 0); \ - } else { \ - assert(atomic_load_32(_lock) >= 0); \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ - } \ -} while (0) +#define SCH_LOCK(type, _lock) \ + do { \ + if (SCH_READ == (type)) { \ + assert(atomic_load_32(_lock) >= 0); \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) > 0); \ + } else { \ + assert(atomic_load_32(_lock) >= 0); \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ + } \ + } while (0) -#define SCH_UNLOCK(type, _lock) do { \ - if (SCH_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ - } else { \ - assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ - } \ -} while (0) +#define SCH_UNLOCK(type, _lock) \ + do { \ + if (SCH_READ == (type)) { \ + assert(atomic_load_32((_lock)) > 0); \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + } else { \ + assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + } \ + } while (0) - -void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); -void schCleanClusterHb(void* pTrans); -int32_t schLaunchTask(SSchJob *job, SSchTask *task); -int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); +void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); +void schCleanClusterHb(void *pTrans); +int32_t schLaunchTask(SSchJob *job, SSchTask *task); +int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); -int32_t schReleaseJob(int64_t refId); -void schFreeFlowCtrl(SSchJob *pJob); -int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); -int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); -int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); -int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); -int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); -int32_t schLaunchFetchTask(SSchJob *pJob); -int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); -int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction); -int32_t schCloneSMsgSendInfo(void *src, void **dst); -int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob); -void schFreeJobImpl(void *job); -int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx); -int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask); -int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans); -int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code); -void schFreeRpcCtx(SRpcCtx *pCtx); -int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp); -bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); -int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); -int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp); -int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); -void schProcessOnDataFetched(SSchJob *job); -int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); -void schFreeRpcCtxVal(const void *arg); -int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); -int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execId); -int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync); -int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus); -int32_t schCancelJob(SSchJob *pJob); -int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); +int32_t schReleaseJob(int64_t refId); +void schFreeFlowCtrl(SSchJob *pJob); +int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); +int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); +int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); +int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); +int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); +int32_t schLaunchFetchTask(SSchJob *pJob); +int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction); +int32_t schCloneSMsgSendInfo(void *src, void **dst); +int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob); +void schFreeJobImpl(void *job); +int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx); +int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask); +int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans); +int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code); +void schFreeRpcCtx(SRpcCtx *pCtx); +int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp); +bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); +int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); +int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp); +int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); +void schProcessOnDataFetched(SSchJob *job); +int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); +void schFreeRpcCtxVal(const void *arg); +int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); +int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execId); +int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync); +int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus); +int32_t schCancelJob(SSchJob *pJob); +int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); uint64_t schGenTaskId(void); -void schCloseJobRef(void); -int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob); -int32_t schJobFetchRows(SSchJob *pJob); -int32_t schJobFetchRowsA(SSchJob *pJob); -int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); -int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); -char* schGetOpStr(SCH_OP_TYPE type); -int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); -int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); -int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); -int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes); -int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet* pEpSet); -int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode); -void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode); -int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq); -void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode); -int32_t schProcessOnCbBegin(SSchJob** job, SSchTask** task, uint64_t qId, int64_t rId, uint64_t tId); -void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask); -bool schJobDone(SSchJob *pJob); -int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask); -int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask); -int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param); -int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq); -int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode); -int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask); -void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode); -int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry); -int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode); -int32_t schProcessOnJobPartialSuccess(SSchJob *pJob); -void schFreeTask(SSchJob *pJob, SSchTask *pTask); -void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); -int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); -int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); -int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); -int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); -void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode); -int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); -int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); -bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); -int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); -int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp); +void schCloseJobRef(void); +int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob); +int32_t schJobFetchRows(SSchJob *pJob); +int32_t schJobFetchRowsA(SSchJob *pJob); +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); +int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); +char *schGetOpStr(SCH_OP_TYPE type); +int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); +int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); +int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); +int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); +int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet); +int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); +void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); +int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq); +void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode); +int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_t rId, uint64_t tId); +void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask); +bool schJobDone(SSchJob *pJob); +int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask); +int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask); +int32_t schSwitchJobStatus(SSchJob *pJob, int32_t status, void *param); +int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob **job, SCH_OP_TYPE type, SSchedulerReq *pReq); +int32_t schHandleOpEndEvent(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); +int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask); +void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode); +int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry); +int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode); +int32_t schProcessOnJobPartialSuccess(SSchJob *pJob); +void schFreeTask(SSchJob *pJob, SSchTask *pTask); +void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); +int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); +int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); +int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); +int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); +void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode); +int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); +int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); +bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); +int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); +int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp); extern SSchDebug gSCHDebug; - #ifdef __cplusplus } #endif diff --git a/source/libs/scheduler/src/schDbg.c b/source/libs/scheduler/src/schDbg.c index a6398522d3..d6c2b638b8 100644 --- a/source/libs/scheduler/src/schDbg.c +++ b/source/libs/scheduler/src/schDbg.c @@ -16,19 +16,17 @@ #include "query.h" #include "schInt.h" -tsem_t schdRspSem; +tsem_t schdRspSem; SSchDebug gSCHDebug = {0}; void schdExecCallback(SExecResult* pResult, void* param, int32_t code) { if (code) { pResult->code = code; } - + *(SExecResult*)param = *pResult; taosMemoryFree(pResult); tsem_post(&schdRspSem); } - - diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index c5c2bfb2bb..8980d08e89 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ +#include "catalog.h" +#include "query.h" #include "schInt.h" #include "tmsg.h" -#include "query.h" -#include "catalog.h" #include "tref.h" void schFreeFlowCtrl(SSchJob *pJob) { @@ -25,14 +25,14 @@ void schFreeFlowCtrl(SSchJob *pJob) { } SSchFlowControl *ctrl = NULL; - void *pIter = taosHashIterate(pJob->flowCtrl, NULL); + void *pIter = taosHashIterate(pJob->flowCtrl, NULL); while (pIter) { ctrl = (SSchFlowControl *)pIter; if (ctrl->taskList) { taosArrayDestroy(ctrl->taskList); } - + pIter = taosHashIterate(pJob->flowCtrl, pIter); } @@ -59,7 +59,8 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { return TSDB_CODE_SUCCESS; } - pJob->flowCtrl = taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pJob->flowCtrl = + taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == pJob->flowCtrl) { SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -73,17 +74,17 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { } int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { - SSchLevel *pLevel = pTask->level; + SSchLevel *pLevel = pTask->level; SSchFlowControl *ctrl = NULL; - int32_t code = 0; - SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - + int32_t code = 0; + SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); + ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_LOCK(SCH_WRITE, &ctrl->lock); if (ctrl->execTaskNum <= 0) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); @@ -93,8 +94,8 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { --ctrl->execTaskNum; ctrl->tableNumSum -= pTask->plan->execNodeStat.tableNum; - SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); _return: @@ -104,11 +105,11 @@ _return: } int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { - SSchLevel *pLevel = pTask->level; - int32_t code = 0; + SSchLevel *pLevel = pTask->level; + int32_t code = 0; SSchFlowControl *ctrl = NULL; - SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - + SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); + do { ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { @@ -119,34 +120,34 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { if (HASH_NODE_EXIST(code)) { continue; } - + SCH_TASK_ELOG("taosHashPut flowCtrl failed, size:%d", (int32_t)sizeof(nctrl)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, nctrl.tableNumSum, nctrl.execTaskNum); + SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, nctrl.tableNumSum, nctrl.execTaskNum); *enough = true; return TSDB_CODE_SUCCESS; } SCH_LOCK(SCH_WRITE, &ctrl->lock); - + if (0 == ctrl->execTaskNum) { ctrl->tableNumSum = pTask->plan->execNodeStat.tableNum; ++ctrl->execTaskNum; - + *enough = true; break; } - + int32_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum; - + if (sum <= schMgmt.cfg.maxNodeTableNum) { ctrl->tableNumSum = sum; ++ctrl->execTaskNum; - + *enough = true; break; } @@ -166,24 +167,25 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { *enough = false; ctrl->sorted = false; - + break; } while (true); _return: - SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ((*enough)?"":"NOT"), ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", + ((*enough) ? "" : "NOT"), ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, + ctrl->execTaskNum); SCH_UNLOCK(SCH_WRITE, &ctrl->lock); - + SCH_RET(code); } -int32_t schTaskTableNumCompare(const void* key1, const void* key2) { +int32_t schTaskTableNumCompare(const void *key1, const void *key2) { SSchTask *pTask1 = *(SSchTask **)key1; SSchTask *pTask2 = *(SSchTask **)key2; - + if (pTask1->plan->execNodeStat.tableNum < pTask2->plan->execNodeStat.tableNum) { return 1; } else if (pTask1->plan->execNodeStat.tableNum > pTask2->plan->execNodeStat.tableNum) { @@ -193,22 +195,21 @@ int32_t schTaskTableNumCompare(const void* key1, const void* key2) { } } - int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SCH_LOCK(SCH_WRITE, &ctrl->lock); - + if (NULL == ctrl->taskList || taosArrayGetSize(ctrl->taskList) <= 0) { SCH_UNLOCK(SCH_WRITE, &ctrl->lock); return TSDB_CODE_SUCCESS; } - int32_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; - int32_t taskNum = taosArrayGetSize(ctrl->taskList); - int32_t code = 0; + int32_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; + int32_t taskNum = taosArrayGetSize(ctrl->taskList); + int32_t code = 0; SSchTask *pTask = NULL; - + if (taskNum > 1 && !ctrl->sorted) { - taosArraySort(ctrl->taskList, schTaskTableNumCompare); // desc order + taosArraySort(ctrl->taskList, schTaskTableNumCompare); // desc order } for (int32_t i = 0; i < taskNum; ++i) { @@ -216,36 +217,36 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) { - SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); continue; } - + ctrl->tableNumSum += pTask->plan->execNodeStat.tableNum; ++ctrl->execTaskNum; taosArrayRemove(ctrl->taskList, i); - - SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); - + + SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); - + remainNum -= pTask->plan->execNodeStat.tableNum; if (remainNum <= 0) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum); - + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, ep->port, + ctrl->tableNumSum, ctrl->execTaskNum); + break; } if (i < (taskNum - 1)) { SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); if (remainNum < pLastTask->plan->execNodeStat.tableNum) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d, smallestInList:%d", - ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); - + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d, smallestInList:%d", + ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); + break; } } @@ -253,7 +254,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { --i; --taskNum; } - + _return: SCH_UNLOCK(SCH_WRITE, &ctrl->lock); @@ -261,11 +262,10 @@ _return: if (code) { code = schProcessOnTaskFailure(pJob, pTask, code); } - + SCH_RET(code); } - int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { if (!SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { return TSDB_CODE_SUCCESS; @@ -274,17 +274,16 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask)); SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - + SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - - int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl);; + + int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl); + ; SCH_ERR_RET(code); - - return code; // to avoid compiler error + + return code; // to avoid compiler error } - - diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 69495c8b7a..bb8451ba37 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -52,9 +52,8 @@ _return: bool schJobDone(SSchJob *pJob) { int8_t status = SCH_GET_JOB_STATUS(pJob); - - return (status == JOB_TASK_STATUS_FAIL || status == JOB_TASK_STATUS_DROP || - status == JOB_TASK_STATUS_SUCC); + + return (status == JOB_TASK_STATUS_FAIL || status == JOB_TASK_STATUS_DROP || status == JOB_TASK_STATUS_SUCC); } FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { @@ -221,7 +220,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); + SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); } SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); @@ -235,7 +234,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - SSchTask* pTask = taosArrayGet(pLevel->subTasks, 0); + SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0); if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType) { pJob->attr.needFetch = true; } @@ -244,7 +243,6 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { return TSDB_CODE_SUCCESS; } - int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) { if (!SCH_IS_DATA_BIND_QRY_TASK(pTask)) { return TSDB_CODE_SUCCESS; @@ -255,7 +253,6 @@ int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { int32_t code = 0; pJob->queryId = pDag->queryId; @@ -365,7 +362,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); _return: - + if (planToTask) { taosHashCleanup(planToTask); } @@ -373,8 +370,7 @@ _return: SCH_RET(code); } - -int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes) { +int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; pRes->res = pJob->execRes.res; @@ -387,13 +383,13 @@ int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes) { return TSDB_CODE_SUCCESS; } -int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { +int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { int32_t code = 0; - + SCH_LOCK(SCH_WRITE, &pJob->resLock); pJob->fetched = true; - + if (pJob->fetchRes && ((SRetrieveTableRsp *)pJob->fetchRes)->completed) { SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_SUCC, NULL)); } @@ -422,12 +418,12 @@ int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { _return: SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - + return code; } -int32_t schNotifyUserExecRes(SSchJob* pJob) { - SExecResult* pRes = taosMemoryCalloc(1, sizeof(SExecResult)); +int32_t schNotifyUserExecRes(SSchJob *pJob) { + SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult)); if (pRes) { schDumpJobExecRes(pJob, pRes); } @@ -439,9 +435,9 @@ int32_t schNotifyUserExecRes(SSchJob* pJob) { return TSDB_CODE_SUCCESS; } -int32_t schNotifyUserFetchRes(SSchJob* pJob) { - void* pRes = NULL; - +int32_t schNotifyUserFetchRes(SSchJob *pJob) { + void *pRes = NULL; + schDumpJobFetchRes(pJob, &pRes); SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode)); @@ -453,17 +449,17 @@ int32_t schNotifyUserFetchRes(SSchJob* pJob) { void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); - + if (SCH_OP_NULL == pJob->opStatus.op) { SCH_JOB_DLOG("job not in any operation, no need to post job res, status:%s", jobTaskStatusStr(pJob->status)); goto _return; } - + if (op && pJob->opStatus.op != op) { SCH_JOB_ELOG("job in operation %s mis-match with expected %s", schGetOpStr(pJob->opStatus.op), schGetOpStr(op)); goto _return; } - + if (SCH_JOB_IN_SYNC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); tsem_post(&pJob->rspSem); @@ -487,7 +483,7 @@ _return: int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { schUpdateJobErrCode(pJob, errCode); - + int32_t code = atomic_load_32(&pJob->errCode); if (code) { SCH_JOB_DLOG("job failed with error %s", tstrerror(code)); @@ -507,9 +503,7 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } -int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { - SCH_RET(schProcessOnJobFailure(pJob, errCode)); -} +int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailure(pJob, errCode)); } int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { @@ -520,8 +514,7 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - -int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { +int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, -1)) { SCH_ERR_RET(schLaunchFetchTask(pJob)); } else { @@ -531,9 +524,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } -void schProcessOnDataFetched(SSchJob *pJob) { - schPostJobRes(pJob, SCH_OP_FETCH); -} +void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); } int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); @@ -548,14 +539,13 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs return TSDB_CODE_SUCCESS; } - int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { if (!SCH_IS_QUERY_JOB(pJob)) { return TSDB_CODE_SUCCESS; } SSchLevel *pLevel = pTask->level; - int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1); + int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1); if (doneNum == pLevel->taskNum) { pJob->levelIdx--; @@ -566,7 +556,7 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { if (pTask->children && taosArrayGetSize(pTask->children) > 0) { continue; } - + SCH_ERR_RET(schLaunchTask(pJob, pTask)); } } @@ -577,11 +567,11 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) { if (rsp->tbFName[0]) { SCH_LOCK(SCH_WRITE, &pJob->resLock); - + if (NULL == pJob->execRes.res) { pJob->execRes.res = taosArrayInit(pJob->taskNum, sizeof(STbVerInfo)); if (NULL == pJob->execRes.res) { - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -610,7 +600,6 @@ int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask) { return TSDB_CODE_SUCCESS; } - int32_t schLaunchJob(SSchJob *pJob) { if (EXPLAIN_MODE_STATIC == pJob->attr.explainMode) { SCH_ERR_RET(qExecStaticExplain(pJob->pDag, (SRetrieveTableRsp **)&pJob->fetchRes)); @@ -623,11 +612,10 @@ int32_t schLaunchJob(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->execTasks); -// schDropTaskInHashList(pJob, pJob->succTasks); -// schDropTaskInHashList(pJob, pJob->failTasks); + // schDropTaskInHashList(pJob, pJob->succTasks); + // schDropTaskInHashList(pJob, pJob->failTasks); } void schFreeJobImpl(void *job) { @@ -659,10 +647,10 @@ void schFreeJobImpl(void *job) { schFreeFlowCtrl(pJob); taosHashCleanup(pJob->execTasks); -// taosHashCleanup(pJob->failTasks); -// taosHashCleanup(pJob->succTasks); + // taosHashCleanup(pJob->failTasks); + // taosHashCleanup(pJob->succTasks); taosHashCleanup(pJob->taskList); - + taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); taosArrayDestroy(pJob->dataSrcTasks); @@ -688,19 +676,19 @@ void schFreeJobImpl(void *job) { } int32_t schJobFetchRows(SSchJob *pJob) { - int32_t code = 0; + int32_t code = 0; if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) { SCH_ERR_RET(schLaunchFetchTask(pJob)); - + if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); - SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); + SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } } else { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { - SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); + SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } else { schPostJobRes(pJob, SCH_OP_FETCH); } @@ -736,9 +724,9 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { } else { pJob->nodeList = taosArrayDup(pReq->pNodeList); } - - pJob->taskList = - taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + + pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, + HASH_ENTRY_LOCK); if (NULL == pJob->taskList) { SCH_JOB_ELOG("taosHashInit %d taskList failed", pReq->pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -750,8 +738,8 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SCH_ERR_JRET(qExecExplainBegin(pReq->pDag, &pJob->explainCtx, pReq->startTs)); } - pJob->execTasks = - taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->execTasks = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, + HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pReq->pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -769,7 +757,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { *pJobId = pJob->refId; - SCH_JOB_DLOG("job refId:0x%" PRIx64" created", pJob->refId); + SCH_JOB_DLOG("job refId:0x%" PRIx64 " created", pJob->refId); return TSDB_CODE_SUCCESS; @@ -782,31 +770,31 @@ _return: } else { taosRemoveRef(schMgmt.jobRef, pJob->refId); } - + SCH_RET(code); } int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { int32_t code = 0; - qDebug("QID:0x%" PRIx64 " sch job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId); + qDebug("QID:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); SCH_ERR_RET(schLaunchJob(pJob)); - + if (pReq->syncReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); } SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); - + return TSDB_CODE_SUCCESS; } -void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode) { +void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode) { if (NULL == pReq || pReq->syncReq) { return; } - + if (pReq->execFp) { (*pReq->execFp)(NULL, pReq->cbParam, errCode); } else if (pReq->fetchFp) { @@ -827,16 +815,17 @@ bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) { return r; } -void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode) { +void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode) { int32_t op = 0; - + switch (type) { case SCH_OP_EXEC: if (pReq && pReq->syncReq) { SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); op = atomic_val_compare_exchange_32(&pJob->opStatus.op, type, SCH_OP_NULL); if (SCH_OP_NULL == op || op != type) { - SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), jobTaskStatusStr(pJob->status)); + SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), + jobTaskStatusStr(pJob->status)); } SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); schDumpJobExecRes(pJob, pReq->pExecRes); @@ -847,7 +836,8 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); op = atomic_val_compare_exchange_32(&pJob->opStatus.op, type, SCH_OP_NULL); if (SCH_OP_NULL == op || op != type) { - SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), jobTaskStatusStr(pJob->status)); + SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), + jobTaskStatusStr(pJob->status)); } SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); } @@ -866,10 +856,10 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode)); } -int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq) { +int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq) { int32_t code = 0; - int8_t status = SCH_GET_JOB_STATUS(pJob); - + int8_t status = SCH_GET_JOB_STATUS(pJob); + switch (type) { case SCH_OP_EXEC: SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); @@ -879,9 +869,9 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - + SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); - + pJob->opStatus.syncReq = pReq->syncReq; SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); break; @@ -893,16 +883,16 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - + SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); - + pJob->userRes.fetchRes = pReq->pFetchRes; pJob->userRes.fetchFp = pReq->fetchFp; pJob->userRes.cbParam = pReq->cbParam; - + pJob->opStatus.syncReq = pReq->syncReq; SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - + if (!SCH_JOB_NEED_FETCH(pJob)) { SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -912,7 +902,7 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + break; case SCH_OP_GET_STATUS: if (pJob->status < JOB_TASK_STATUS_INIT || pJob->levelNum <= 0 || NULL == pJob->levels) { @@ -941,23 +931,23 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { if (errCode) { schHandleJobFailure(pJob, errCode); } - + if (pJob) { schReleaseJob(pJob->refId); } } -int32_t schProcessOnCbBegin(SSchJob** job, SSchTask** task, uint64_t qId, int64_t rId, uint64_t tId) { +int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_t rId, uint64_t tId) { int32_t code = 0; - int8_t status = 0; + int8_t status = 0; SSchTask *pTask = NULL; - SSchJob *pJob = schAcquireJob(rId); + SSchJob *pJob = schAcquireJob(rId); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); } - + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); @@ -980,9 +970,6 @@ _return: if (pJob) { schReleaseJob(rId); } - + SCH_RET(code); } - - - diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index 64cda573f1..66d58c95d1 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -29,25 +29,25 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { case JOB_TASK_STATUS_INIT: break; case JOB_TASK_STATUS_EXEC: - SCH_ERR_JRET(schExecJob(pJob, (SSchedulerReq*)param)); + SCH_ERR_JRET(schExecJob(pJob, (SSchedulerReq*)param)); break; case JOB_TASK_STATUS_PART_SUCC: SCH_ERR_JRET(schProcessOnJobPartialSuccess(pJob)); break; case JOB_TASK_STATUS_SUCC: break; - case JOB_TASK_STATUS_FAIL: + case JOB_TASK_STATUS_FAIL: SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0))); break; case JOB_TASK_STATUS_DROP: schProcessOnJobDropped(pJob, *(int32_t*)param); - + if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) { SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId); } else { SCH_JOB_DLOG("job removed from jobRef list, refId:0x%" PRIx64, pJob->refId); } - break; + break; default: { SCH_JOB_ELOG("unknown job status %d", status); SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); @@ -62,7 +62,7 @@ _return: } int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { - SSchJob *pJob = schAcquireJob(jobId); + SSchJob* pJob = schAcquireJob(jobId); if (NULL == pJob) { qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); @@ -75,12 +75,12 @@ int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SS int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode) { int32_t code = errCode; - + if (NULL == pJob) { schDirectPostJobRes(pReq, errCode); SCH_RET(code); } - + schProcessOnOpEnd(pJob, type, pReq, errCode); if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { @@ -91,5 +91,3 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq return code; } - - diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index b6f96a188e..33bf50d853 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -16,12 +16,12 @@ #include "catalog.h" #include "command.h" #include "query.h" +#include "qworker.h" #include "schInt.h" +#include "tglobal.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" -#include "qworker.h" -#include "tglobal.h" void schFreeTask(SSchJob *pJob, SSchTask *pTask) { schDeregisterTaskHb(pJob, pTask); @@ -94,7 +94,7 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_LOCAL_EXEC_TASK(pJob, pTask)) { return TSDB_CODE_SUCCESS; } - + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx, @@ -162,14 +162,15 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v if (dropExecNode) { SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } - + schUpdateTaskExecNode(pJob, pTask, handle, execId); if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it - SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, + pTask->execId, pTask->waitRetry); SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); } - + SCH_SET_TASK_HANDLE(pTask, handle); return TSDB_CODE_SUCCESS; @@ -837,17 +838,18 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { goto _return; } - SSchTask *pTask = NULL; - SSchJob *pJob = NULL; + SSchTask *pTask = NULL; + SSchJob *pJob = NULL; for (int32_t i = 0; i < resNum; ++i) { - SExplainLocalRsp* localRsp = taosArrayGet(pExplainRes, i); + SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); pJob = schAcquireJob(localRsp->rId); if (NULL == pJob) { - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, + localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -857,16 +859,17 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { schReleaseJob(pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } - + code = schGetTaskInJob(pJob, localRsp->tId, &pTask); if (TSDB_CODE_SUCCESS == code) { code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp); } - + schReleaseJob(pJob->refId); - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, + localRsp->tId, code); SCH_ERR_JRET(code); @@ -879,7 +882,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { _return: for (int32_t i = 0; i < resNum; ++i) { - SExplainLocalRsp* localRsp = taosArrayGet(pExplainRes, i); + SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); tFreeSExplainRsp(&localRsp->rsp); } @@ -890,7 +893,7 @@ _return: int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { SSubplan *plan = pTask->plan; - int32_t code = 0; + int32_t code = 0; if (NULL == pTask->msg) { // TODO add more detailed reason for failure code = qSubPlanToMsg(plan, &pTask->msg, &pTask->msgLen); @@ -899,7 +902,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { pTask->msgLen); SCH_ERR_RET(code); } else if (tsQueryPlannerTrace) { - char *msg = NULL; + char *msg = NULL; int32_t msgLen = 0; qSubPlanToString(plan, &msg, &msgLen); SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); @@ -910,20 +913,20 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); +// SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } - + SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); } int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { - //SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); + // SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); if (NULL == schMgmt.queryMgmt) { SCH_ERR_RET(qWorkerInit(NODE_TYPE_CLIENT, CLIENT_HANDLE, (void **)&schMgmt.queryMgmt, NULL)); } SArray *explainRes = NULL; - SQWMsg qwMsg = {0}; + SQWMsg qwMsg = {0}; qwMsg.msgInfo.taskType = TASK_TYPE_TEMP; qwMsg.msgInfo.explain = SCH_IS_EXPLAIN_JOB(pJob); qwMsg.msgInfo.needFetch = SCH_TASK_NEED_FETCH(pTask); @@ -934,8 +937,9 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - - SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &qwMsg, explainRes)); + + SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -958,17 +962,17 @@ int32_t schLaunchTaskImpl(void *param) { if (pCtx->asyncLaunch) { SCH_LOCK_TASK(pTask); } - - int8_t status = 0; - int32_t code = 0; + + int8_t status = 0; + int32_t code = 0; atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; pTask->waitRetry = false; - SCH_TASK_DLOG("start to launch %s task, execId %d, retry %d", SCH_IS_LOCAL_EXEC_TASK(pJob, pTask) ? "LOCAL" : "REMOTE", - pTask->execId, pTask->retryTimes); + SCH_TASK_DLOG("start to launch %s task, execId %d, retry %d", + SCH_IS_LOCAL_EXEC_TASK(pJob, pTask) ? "LOCAL" : "REMOTE", pTask->execId, pTask->retryTimes); SCH_LOG_TASK_START_TS(pTask); @@ -989,6 +993,12 @@ int32_t schLaunchTaskImpl(void *param) { SCH_ERR_JRET(schLaunchRemoteTask(pJob, pTask)); } +#if 0 + if (SCH_IS_QUERY_JOB(pJob)) { + SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask)); + } +#endif + _return: if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { @@ -1086,19 +1096,20 @@ int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) { } int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { - void *pRsp = NULL; + void *pRsp = NULL; SArray *explainRes = NULL; if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &pRsp, explainRes)); + SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); } - + SCH_ERR_RET(schProcessFetchRsp(pJob, pTask, pRsp, TSDB_CODE_SUCCESS)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 6f12780ff9..7099dbb2c0 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -21,21 +21,21 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { - qDebug("sch acquire jobId:0x%"PRIx64, refId); - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { + qDebug("sch acquire jobId:0x%" PRIx64, refId); + return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } -FORCE_INLINE int32_t schReleaseJob(int64_t refId) { +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { if (0 == refId) { return TSDB_CODE_SUCCESS; } - - qDebug("sch release jobId:0x%"PRIx64, refId); - return taosReleaseRef(schMgmt.jobRef, refId); + + qDebug("sch release jobId:0x%" PRIx64, refId); + return taosReleaseRef(schMgmt.jobRef, refId); } -char* schGetOpStr(SCH_OP_TYPE type) { +char *schGetOpStr(SCH_OP_TYPE type) { switch (type) { case SCH_OP_NULL: return "NULL"; @@ -53,28 +53,28 @@ char* schGetOpStr(SCH_OP_TYPE type) { void schFreeHbTrans(SSchHbTrans *pTrans) { rpcReleaseHandle(pTrans->trans.pHandle, TAOS_CONN_CLIENT); - schFreeRpcCtx(&pTrans->rpcCtx); + schFreeRpcCtx(&pTrans->rpcCtx); } -void schCleanClusterHb(void* pTrans) { +void schCleanClusterHb(void *pTrans) { SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); SSchHbTrans *hb = taosHashIterate(schMgmt.hbConnections, NULL); while (hb) { if (hb->trans.pTrans == pTrans) { - SQueryNodeEpId* pEpId = taosHashGetKey(hb, NULL); + SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL); schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } - + hb = taosHashIterate(schMgmt.hbConnections, hb); } - + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); } int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { - int32_t code = 0; + int32_t code = 0; SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); @@ -94,7 +94,6 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep return TSDB_CODE_SUCCESS; } - int32_t schAddHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { int32_t code = 0; SSchHbTrans hb = {0}; @@ -155,13 +154,13 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { if (!pTask->registerdHb) { return; } - + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - SEp* pEp = SCH_GET_CUR_EP(addr); + SEp *pEp = SCH_GET_CUR_EP(addr); strcpy(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; @@ -180,24 +179,22 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { } else { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); } - + pTask->registerdHb = false; } - - int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - SEp* pEp = SCH_GET_CUR_EP(addr); + SEp *pEp = SCH_GET_CUR_EP(addr); strcpy(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); - + pTask->registerdHb = true; return TSDB_CODE_SUCCESS; @@ -226,7 +223,6 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { return TSDB_CODE_SUCCESS; } - void schCloseJobRef(void) { if (!atomic_load_8((int8_t *)&schMgmt.exit)) { return; @@ -242,7 +238,7 @@ uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } uint64_t schGenUUID(void) { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -254,15 +250,14 @@ uint64_t schGenUUID(void) { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } - void schFreeRpcCtxVal(const void *arg) { if (NULL == arg) { return; @@ -307,5 +302,3 @@ int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTas return TSDB_CODE_SUCCESS; } - - diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 0ccaef3857..c4b45649a4 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -14,10 +14,10 @@ */ #include "query.h" +#include "qworker.h" #include "schInt.h" #include "tmsg.h" #include "tref.h" -#include "qworker.h" SSchedulerMgmt schMgmt = { .jobRef = -1, @@ -35,7 +35,7 @@ int32_t schedulerInit() { schMgmt.cfg.enableReSchedule = true; qDebug("schedule policy init to %d", schMgmt.cfg.schPolicy); - + schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); @@ -61,7 +61,7 @@ int32_t schedulerInit() { int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId) { qDebug("scheduler %s exec job start", pReq->syncReq ? "SYNC" : "ASYNC"); - int32_t code = 0; + int32_t code = 0; SSchJob *pJob = NULL; SCH_ERR_JRET(schInitJob(pJobId, pReq)); @@ -73,7 +73,7 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId) { SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_EXEC, pReq)); _return: - + SCH_RET(schHandleOpEndEvent(pJob, SCH_OP_EXEC, pReq, code)); } @@ -144,7 +144,7 @@ int32_t schedulerEnableReSchedule(bool enableResche) { return TSDB_CODE_SUCCESS; } -void schedulerFreeJob(int64_t* jobId, int32_t errCode) { +void schedulerFreeJob(int64_t *jobId, int32_t errCode) { if (0 == *jobId) { return; } @@ -155,10 +155,9 @@ void schedulerFreeJob(int64_t* jobId, int32_t errCode) { return; } - SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", errCode:0x%x", *jobId, errCode); - + SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); schHandleJobDrop(pJob, errCode); - + schReleaseJob(*jobId); *jobId = 0; } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index ca2122ed8f..97e14b617c 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -30,15 +30,12 @@ #endif #include "os.h" -#include "tglobal.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" #include "catalog.h" #include "scheduler.h" #include "taos.h" #include "tdatablock.h" #include "tdef.h" +#include "tglobal.h" #include "trpc.h" #include "tvariant.h" @@ -56,8 +53,9 @@ namespace { -extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); -extern "C" int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode); +extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, + int32_t rspCode); +extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode); int64_t insertJobRefId = 0; int64_t queryJobRefId = 0; @@ -66,16 +64,15 @@ uint64_t schtMergeTemplateId = 0x4; uint64_t schtFetchTaskId = 0; uint64_t schtQueryId = 1; -bool schtTestStop = false; -bool schtTestDeadLoop = false; +bool schtTestStop = false; +bool schtTestDeadLoop = false; int32_t schtTestMTRunSec = 10; int32_t schtTestPrintNum = 1000; int32_t schtStartFetch = 0; - void schtInitLogFile() { - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; @@ -84,24 +81,22 @@ void schtInitLogFile() { if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } - } -void schtQueryCb(SExecResult* pResult, void* param, int32_t code) { +void schtQueryCb(SExecResult *pResult, void *param, int32_t code) { assert(TSDB_CODE_SUCCESS == code); - *(int32_t*)param = 1; + *(int32_t *)param = 1; } - void schtBuildQueryDag(SQueryPlan *dag) { uint64_t qId = schtQueryId; - + dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); - SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - + SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); @@ -117,7 +112,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); - scanPlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + scanPlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan->msgType = TDMT_SCH_QUERY; mergePlan->id.queryId = qId; @@ -129,32 +124,32 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->pChildren = nodesMakeList(); mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_SCH_QUERY; merge->pNodeList = nodesMakeList(); scan->pNodeList = nodesMakeList(); - nodesListAppend(merge->pNodeList, (SNode*)mergePlan); - nodesListAppend(scan->pNodeList, (SNode*)scanPlan); + nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + nodesListAppend(scan->pNodeList, (SNode *)scanPlan); - nodesListAppend(mergePlan->pChildren, (SNode*)scanPlan); - nodesListAppend(scanPlan->pParents, (SNode*)mergePlan); + nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode*)merge); - nodesListAppend(dag->pSubplans, (SNode*)scan); + nodesListAppend(dag->pSubplans, (SNode *)merge); + nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { uint64_t qId = schtQueryId; - int32_t scanPlanNum = 20; - + int32_t scanPlanNum = 20; + dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); - SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - + SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(scanPlanNum, sizeof(SSubplan)); SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); @@ -180,13 +175,13 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan[i].pChildren = NULL; scanPlan[i].level = 1; scanPlan[i].pParents = nodesMakeList(); - scanPlan[i].pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + scanPlan[i].pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan[i].msgType = TDMT_SCH_QUERY; - nodesListAppend(scanPlan[i].pParents, (SNode*)mergePlan); - nodesListAppend(mergePlan->pChildren, (SNode*)(scanPlan + i)); + nodesListAppend(scanPlan[i].pParents, (SNode *)mergePlan); + nodesListAppend(mergePlan->pChildren, (SNode *)(scanPlan + i)); - nodesListAppend(scan->pNodeList, (SNode*)(scanPlan + i)); + nodesListAppend(scan->pNodeList, (SNode *)(scanPlan + i)); } mergePlan->id.queryId = qId; @@ -197,29 +192,25 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(merge->pNodeList, (SNode*)mergePlan); - - nodesListAppend(dag->pSubplans, (SNode*)merge); - nodesListAppend(dag->pSubplans, (SNode*)scan); -} - - -void schtFreeQueryDag(SQueryPlan *dag) { + nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + nodesListAppend(dag->pSubplans, (SNode *)merge); + nodesListAppend(dag->pSubplans, (SNode *)scan); } +void schtFreeQueryDag(SQueryPlan *dag) {} void schtBuildInsertDag(SQueryPlan *dag) { uint64_t qId = 0x0000000000000002; - + dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); - SNodeListNode *inserta = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - + SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SSubplan *insertPlan = (SSubplan *)taosMemoryCalloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; @@ -235,7 +226,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; - insertPlan[0].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); + insertPlan[0].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; @@ -251,48 +242,43 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; - insertPlan[1].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); + insertPlan[1].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[1].msgType = TDMT_VND_SUBMIT; inserta->pNodeList = nodesMakeList(); - nodesListAppend(inserta->pNodeList, (SNode*)insertPlan); + nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); insertPlan += 1; - nodesListAppend(inserta->pNodeList, (SNode*)insertPlan); + nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); - nodesListAppend(dag->pSubplans, (SNode*)inserta); + nodesListAppend(dag->pSubplans, (SNode *)inserta); } - -int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { +int32_t schtPlanToString(const SSubplan *subplan, char **str, int32_t *len) { *str = (char *)taosMemoryCalloc(1, 20); *len = 20; return 0; } -void schtExecNode(SSubplan* subplan, uint64_t groupId, SQueryNodeAddr* ep) { +void schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) {} -} - -void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { - -} +void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {} void schtSetPlanToString() { static Stub stub; stub.set(qSubPlanToString, schtPlanToString); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("qSubPlanToString", result); #endif #ifdef LINUX - AddrAny any("libplanner.so"); - std::map result; + AddrAny any("libplanner.so"); + std::map result; any.get_global_func_addr_dynsym("^qSubPlanToString$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtPlanToString); } } @@ -303,16 +289,16 @@ void schtSetExecNode() { stub.set(qSetSubplanExecutionNode, schtExecNode); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("qSetSubplanExecutionNode", result); #endif #ifdef LINUX - AddrAny any("libplanner.so"); - std::map result; + AddrAny any("libplanner.so"); + std::map result; any.get_global_func_addr_dynsym("^qSetSubplanExecutionNode$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtExecNode); } } @@ -323,22 +309,22 @@ void schtSetRpcSendRequest() { stub.set(rpcSendRequest, schtRpcSendRequest); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("rpcSendRequest", result); #endif #ifdef LINUX - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRequest$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtRpcSendRequest); } } } -int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { +int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo) { if (pInfo) { taosMemoryFreeClear(pInfo->param); taosMemoryFreeClear(pInfo->msgInfo.pData); @@ -347,32 +333,30 @@ int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTr return 0; } - void schtSetAsyncSendMsgToServer() { static Stub stub; stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("asyncSendMsgToServer", result); #endif #ifdef LINUX - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtAsyncSendMsgToServer); } } } - void *schtSendRsp(void *param) { SSchJob *pJob = NULL; - int64_t job = 0; - int32_t code = 0; + int64_t job = 0; + int32_t code = 0; while (true) { job = *(int64_t *)param; @@ -384,7 +368,7 @@ void *schtSendRsp(void *param) { } pJob = schAcquireJob(job); - + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -392,9 +376,9 @@ void *schtSendRsp(void *param) { SSubmitRsp rsp = {0}; rsp.affectedRows = 10; schHandleResponseMsg(pJob, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); - + pIter = taosHashIterate(pJob->execTasks, pIter); - } + } schReleaseJob(job); @@ -402,28 +386,27 @@ void *schtSendRsp(void *param) { } void *schtCreateFetchRspThread(void *param) { - int64_t job = *(int64_t *)param; - SSchJob* pJob = schAcquireJob(job); + int64_t job = *(int64_t *)param; + SSchJob *pJob = schAcquireJob(job); taosSsleep(1); - int32_t code = 0; + int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; - + code = schHandleResponseMsg(pJob, pJob->fetchTask, TDMT_SCH_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); schReleaseJob(job); - + assert(code == 0); return NULL; } - void *schtFetchRspThread(void *aa) { - SDataBuf dataBuf = {0}; - SSchTaskCallbackParam* param = NULL; + SDataBuf dataBuf = {0}; + SSchTaskCallbackParam *param = NULL; while (!schtTestStop) { if (0 == atomic_val_compare_exchange_32(&schtStartFetch, 1, 0)) { @@ -431,13 +414,13 @@ void *schtFetchRspThread(void *aa) { } taosUsleep(1); - + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); - param->queryId = schtQueryId; + param->queryId = schtQueryId; param->taskId = schtFetchTaskId; - int32_t code = 0; + int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -446,7 +429,7 @@ void *schtFetchRspThread(void *aa) { dataBuf.len = sizeof(*rsp); code = schHandleCallback(param, &dataBuf, TDMT_SCH_FETCH_RSP, 0); - + assert(code == 0 || code); } return NULL; @@ -454,8 +437,8 @@ void *schtFetchRspThread(void *aa) { void schtFreeQueryJob(int32_t freeThread) { static uint32_t freeNum = 0; - int64_t job = queryJobRefId; - + int64_t job = queryJobRefId; + if (job && atomic_val_compare_exchange_64(&queryJobRefId, job, 0)) { schedulerFreeJob(&job, 0); if (freeThread) { @@ -466,31 +449,29 @@ void schtFreeQueryJob(int32_t freeThread) { } } -void* schtRunJobThread(void *aa) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; +void *schtRunJobThread(void *aa) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - SQueryPlan dag; + SQueryPlan dag; schtInitLogFile(); - int32_t code = schedulerInit(); assert(code == 0); - schtSetPlanToString(); schtSetExecNode(); schtSetAsyncSendMsgToServer(); - SSchJob *pJob = NULL; + SSchJob *pJob = NULL; SSchTaskCallbackParam *param = NULL; - SHashObj *execTasks = NULL; - SDataBuf dataBuf = {0}; - uint32_t jobFinished = 0; - int32_t queryDone = 0; + SHashObj *execTasks = NULL; + SDataBuf dataBuf = {0}; + uint32_t jobFinished = 0; + int32_t queryDone = 0; while (!schtTestStop) { schtBuildQueryDag(&dag); @@ -503,10 +484,10 @@ void* schtRunJobThread(void *aa) { taosArrayPush(qnodeList, &qnodeAddr); queryDone = 0; - + SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.syncReq = false; req.pConn = &conn; req.pNodeList = qnodeList; @@ -514,8 +495,8 @@ void* schtRunJobThread(void *aa) { req.sql = "select * from tb"; req.execFp = schtQueryCb; req.cbParam = &queryDone; - - code = schedulerExecJob(&req, &queryJobRefId); + + code = schedulerExecJob(&req, &queryJobRefId); assert(code == 0); pJob = schAcquireJob(queryJobRefId); @@ -524,20 +505,20 @@ void* schtRunJobThread(void *aa) { schtFreeQueryDag(&dag); continue; } - + execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; - + taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; - param->queryId = pJob->queryId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -547,16 +528,16 @@ void* schtRunJobThread(void *aa) { SQueryTableRsp rsp = {0}; dataBuf.pData = &rsp; dataBuf.len = sizeof(rsp); - + code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0); assert(code == 0 || code); pIter = taosHashIterate(execTasks, pIter); - } + } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; - param->queryId = pJob->queryId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -566,28 +547,27 @@ void* schtRunJobThread(void *aa) { SQueryTableRsp rsp = {0}; dataBuf.pData = &rsp; dataBuf.len = sizeof(rsp); - + code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0); assert(code == 0 || code); - - pIter = taosHashIterate(execTasks, pIter); - } + pIter = taosHashIterate(execTasks, pIter); + } while (true) { if (queryDone) { break; } - + taosUsleep(10000); } atomic_store_32(&schtStartFetch, 1); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; - + code = schedulerFetchRows(queryJobRefId, &req); assert(code == 0 || code); @@ -600,7 +580,7 @@ void* schtRunJobThread(void *aa) { data = NULL; code = schedulerFetchRows(queryJobRefId, &req); assert(code == 0 || code); - + schtFreeQueryJob(0); taosHashCleanup(execTasks); @@ -620,7 +600,7 @@ void* schtRunJobThread(void *aa) { return NULL; } -void* schtFreeJobThread(void *aa) { +void *schtFreeJobThread(void *aa) { while (!schtTestStop) { taosUsleep(taosRand() % 100); schtFreeQueryJob(1); @@ -628,17 +608,16 @@ void* schtFreeJobThread(void *aa) { return NULL; } - -} +} // namespace TEST(queryTest, normalCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - int64_t job = 0; - SQueryPlan dag; + int64_t job = 0; + SQueryPlan dag; memset(&dag, 0, sizeof(dag)); @@ -648,7 +627,7 @@ TEST(queryTest, normalCase) { strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -662,30 +641,29 @@ TEST(queryTest, normalCase) { SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "select * from tb"; req.execFp = schtQueryCb; req.cbParam = &queryDone; - - code = schedulerExecJob(&req, &job); + + code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -693,10 +671,10 @@ TEST(queryTest, normalCase) { SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } while (true) { if (queryDone) { @@ -705,14 +683,14 @@ TEST(queryTest, normalCase) { taosUsleep(10000); } - + TdThreadAttr thattr; taosThreadAttrInit(&thattr); TdThread thread1; taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; @@ -739,13 +717,13 @@ TEST(queryTest, normalCase) { } TEST(queryTest, readyFirstCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - int64_t job = 0; - SQueryPlan dag; + int64_t job = 0; + SQueryPlan dag; memset(&dag, 0, sizeof(dag)); @@ -755,7 +733,7 @@ TEST(queryTest, readyFirstCase) { strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -765,11 +743,11 @@ TEST(queryTest, readyFirstCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - int32_t queryDone = 0; + int32_t queryDone = 0; SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; @@ -779,19 +757,18 @@ TEST(queryTest, readyFirstCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -799,10 +776,10 @@ TEST(queryTest, readyFirstCase) { SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } while (true) { if (queryDone) { @@ -812,14 +789,13 @@ TEST(queryTest, readyFirstCase) { taosUsleep(10000); } - TdThreadAttr thattr; taosThreadAttrInit(&thattr); TdThread thread1; taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; code = schedulerFetchRows(job, &req); @@ -844,28 +820,26 @@ TEST(queryTest, readyFirstCase) { schedulerDestroy(); } - - TEST(queryTest, flowCtrlCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - int64_t job = 0; - SQueryPlan dag; + int64_t job = 0; + SQueryPlan dag; schtInitLogFile(); taosSeedRand(taosGetTimestampSec()); - + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -875,10 +849,10 @@ TEST(queryTest, flowCtrlCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - int32_t queryDone = 0; + int32_t queryDone = 0; SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; @@ -889,17 +863,16 @@ TEST(queryTest, flowCtrlCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); bool qDone = false; - + while (!qDone) { void *pIter = taosHashIterate(pJob->execTasks, NULL); if (NULL == pIter) { break; } - + while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -908,15 +881,15 @@ TEST(queryTest, flowCtrlCase) { if (task->lastMsgType == TDMT_SCH_QUERY) { SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); } else { qDone = true; break; } - + pIter = NULL; - } + } } while (true) { @@ -933,7 +906,7 @@ TEST(queryTest, flowCtrlCase) { TdThread thread1; taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; code = schedulerFetchRows(job, &req); @@ -958,15 +931,14 @@ TEST(queryTest, flowCtrlCase) { schedulerDestroy(); } - TEST(insertTest, normalCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - SQueryPlan dag; - uint64_t numOfRows = 0; + SQueryPlan dag; + uint64_t numOfRows = 0; SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); @@ -974,7 +946,7 @@ TEST(insertTest, normalCase) { strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -993,21 +965,21 @@ TEST(insertTest, normalCase) { SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "insert into tb values(now,1)"; req.execFp = schtQueryCb; req.cbParam = NULL; - + code = schedulerExecJob(&req, &insertJobRefId); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); schedulerFreeJob(&insertJobRefId, 0); - schedulerDestroy(); + schedulerDestroy(); } TEST(multiThread, forceFree) { @@ -1027,12 +999,12 @@ TEST(multiThread, forceFree) { break; } } - + schtTestStop = true; taosSsleep(3); } -int main(int argc, char** argv) { +int main(int argc, char **argv) { taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 4a63cd3bb2..ee317d0751 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -67,7 +67,11 @@ void streamSchedByTimer(void* param, void* tmrId) { atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - streamTaskInput(pTask, (SStreamQueueItem*)trigger); + if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) { + taosFreeQitem(trigger); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + return; + } streamSchedExec(pTask); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 63a28e17ed..53e49a6ba5 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -28,6 +28,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF sprintf(streamPath, "%s/%s", path, "stream"); pMeta->path = strdup(streamPath); if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db) < 0) { + taosMemoryFree(streamPath); goto _err; } @@ -58,7 +59,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF return pMeta; _err: - if (pMeta->path) taosMemoryFree(pMeta->path); + taosMemoryFree(pMeta->path); if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks); if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb); @@ -250,6 +251,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { + tdbFree(pKey); + tdbFree(pVal); return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); @@ -257,10 +260,14 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { tDecoderClear(&decoder); if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ea590c57e6..c0ed135d74 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -63,7 +63,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int sprintf(statePath, "%s/%d", path, pTask->taskId); } else { memset(statePath, 0, 300); - strncpy(statePath, path, 300); + tstrncpy(statePath, path, 300); } if (tdbOpen(statePath, szPage, pages, &pState->db) < 0) { goto _err; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 7c4aaba6ce..5aadb599a3 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -174,14 +174,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { if (res == TSDB_CODE_SUCCESS) { return false; } else { - qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 - ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } } - qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } @@ -203,14 +198,10 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { } if (ts < pInfo->minTS) { - qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } else if (res == TSDB_CODE_SUCCESS) { return false; } - qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, - maxTs, *pMapMaxTs, ts); // check from tsdb api return true; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0afc373f2d..101e99afea 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -169,7 +169,7 @@ typedef struct SSyncNode { } SSyncNode; // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e000ba8bf8..4a35a15d3e 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index b604d25816..511113352e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); - ASSERT(pEntry != NULL); + if (pEntry == NULL) { + sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index); + return; + } } // cannot commit, even if quorum agree. need check term! if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { @@ -127,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); - ASSERT(code == 0); + if (code != 0) { + wError("failed to commit sync node since %s", tstrerror(terrno)); + } } } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 378ac743b3..907eb21f4c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -86,10 +86,10 @@ void syncCleanUp() { } } -int64_t syncOpen(const SSyncInfo* pSyncInfo) { +int64_t syncOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); if (pSyncNode == NULL) { - sError("failed to open sync node. vgId:%d", pSyncInfo->vgId); + sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr()); return -1; } @@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg return ret; } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { } ASSERT(rid == pSyncNode->rid); +#if 0 if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; @@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; +#else + syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); + syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return 0; +#endif } int32_t syncLeaderTransfer(int64_t rid) { @@ -919,9 +926,7 @@ _END: } // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { - SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; - +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode)); if (pSyncNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { sError("failed to open raft cfg file. path:%s", pSyncNode->configPath); goto _error; } - pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + if (pSyncInfo->syncCfg.replicaNum == 0) { + pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + } raftCfgClose(pSyncNode->pRaftCfg); pSyncNode->pRaftCfg = NULL; @@ -2650,7 +2657,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { if (ths->state == TAOS_SYNC_STATE_LEADER) { int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); + return -1; + } syncNodeReplicate(ths, false); } @@ -2726,7 +2736,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { // del resp mgr, call FpCommitCb - ASSERT(0); + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -2790,8 +2800,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); // del resp mgr, call FpCommitCb - ASSERT(0); return -1; } @@ -3043,7 +3053,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i); + return -1; + } ASSERT(pEntry != NULL); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 496c8419de..c3dad104d1 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); - - ASSERT(0); return -1; } pEntry->index = index; @@ -414,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { return 0; } -SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { +SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { - taosThreadMutexLock(&(pData->mutex)); - // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReader* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); @@ -444,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } while (0); - ASSERT(0); + sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index); + return NULL; } SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); @@ -465,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { terrno = saveErr; */ - taosThreadMutexUnlock(&(pData->mutex)); return pEntry; } else { @@ -473,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } +SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { + SSyncLogStoreData* pData = pLogStore->data; + SSyncRaftEntry *pEntry = NULL; + + taosThreadMutexLock(&pData->mutex); + pEntry = logStoreGetEntryWithoutLock(pLogStore, index); + taosThreadMutexUnlock(&pData->mutex); + return pEntry; +} + int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a148c1e36b..13b7e0ac0a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN ret = tdbPagerWrite(pPager, pChild); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to write page since %s", terrstr()); + return -1; } // Copy the root page content to the child page @@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pOlds[i]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pParent); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) // mark dirty ret = tdbPagerWrite(pBt->pPager, *ppOfp); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -987,7 +984,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch a new ofp and make it dirty SPgno pgno = 0; - SPage *ofp, *nextOfp; + SPage *ofp = NULL, *nextOfp = NULL; ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { @@ -1942,7 +1939,7 @@ int tdbBtcDelete(SBTC *pBtc) { // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1964,7 +1961,7 @@ int tdbBtcDelete(SBTC *pBtc) { if (idx < nCells) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -2029,7 +2026,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // mark dirty ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index ea16e80562..6c01348bc2 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 04711eb6a0..dd2416e5b8 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -34,9 +34,9 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); -static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) { - SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode)); - SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) { + SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node)); + SPage *pPageR = (SPage *)(((uint8_t *)rhs) - offsetof(SPage, node)); SPgno pgnoL = TDB_PAGE_PGNO(pPageL); SPgno pgnoR = TDB_PAGE_PGNO(pPageR); @@ -156,6 +156,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -210,7 +211,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } } @@ -226,6 +227,8 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { // Open the journal pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (pPager->jfd < 0) { + tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -243,9 +246,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } // loop to write the dirty pages to file @@ -255,7 +258,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage = (SPage *)pNode; ret = tdbPagerWritePageToDB(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; } } @@ -277,11 +280,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } // remove the journal file - tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(pPager->jfd) < 0) { + tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pPager->inTran = 0; return 0; @@ -297,14 +314,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 0, sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); if (jfd == NULL) { - return 0; + return -1; } ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); @@ -516,11 +533,15 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -540,13 +561,15 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -590,23 +613,38 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } tdbOsFree(pageBuf); - tdbOsClose(jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(jfd) < 0) { + tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } return 0; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index e5a783292d..c82af0d0e9 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -100,8 +100,8 @@ void rpcCloseImpl(void* arg) { taosMemoryFree(pRpc); } -void* rpcMallocCont(int32_t contLen) { - int32_t size = contLen + TRANS_MSG_OVERHEAD; +void* rpcMallocCont(int64_t contLen) { + int64_t size = contLen + TRANS_MSG_OVERHEAD; char* start = taosMemoryCalloc(1, size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); @@ -120,11 +120,11 @@ void rpcFreeCont(void* cont) { tTrace("rpc free cont:%p", (char*)cont - TRANS_MSG_OVERHEAD); } -void* rpcReallocCont(void* ptr, int32_t contLen) { +void* rpcReallocCont(void* ptr, int64_t contLen) { if (ptr == NULL) return rpcMallocCont(contLen); char* st = (char*)ptr - TRANS_MSG_OVERHEAD; - int32_t sz = contLen + TRANS_MSG_OVERHEAD; + int64_t sz = contLen + TRANS_MSG_OVERHEAD; st = taosMemoryRealloc(st, sz); if (st == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 83323725be..359b9824cf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -597,6 +597,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; + tDebug("%s conn %p alloc read buf", CONN_GET_INST_LABEL(conn), conn); transAllocBuffer(pBuf, buf); } static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { @@ -609,7 +610,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; while (transReadComplete(pBuf)) { - tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); + tDebug("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); if (pBuf->invalid) { cliHandleExcept(conn); break; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 42079ff005..f843c8559c 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -203,7 +203,7 @@ bool transReadComplete(SConnBuffer* connBuf) { } int transSetConnOption(uv_tcp_t* stream) { - uv_tcp_nodelay(stream, 1); + uv_tcp_nodelay(stream, 0); int ret = uv_tcp_keepalive(stream, 5, 60); return ret; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 57bf8a34b3..3616b4808f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -332,7 +332,10 @@ void uvOnSendCb(uv_write_t* req, int status) { if (status == 0) { tTrace("conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { - SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + STraceId* trace = &msg->msg.info.traceId; + tGDebug("conn %p write data out", conn); + destroySmsg(msg); // send cached data if (!transQueueEmpty(&conn->srvMsgs)) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e49a963191..279f4dc656 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -35,113 +35,251 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; } -static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) { - sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { + return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } -static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { - int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(sz > 0); -#if 0 - for (int i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); - } -#endif +static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { + return sprintf(buf, "%s/meta-ver.tmp", pWal->path); +} - SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); +static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + terrno = TSDB_CODE_SUCCESS; + ASSERT(fileIdx >= 0 && fileIdx < sz); + + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); int64_t fileSize = 0; taosStatFile(fnameStr, &fileSize, NULL); - int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); - pLastFileInfo->fileSize = fileSize; - TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + // ensure size as non-negative + pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize); + uint64_t magic = WAL_MAGIC; + int64_t walCkHeadSz = sizeof(SWalCkHead); + int64_t end = fileSize; + int64_t offset = 0; + int32_t capacity = 0; + int32_t readSize = 0; + char* buf = NULL; + char* found = NULL; + bool firstTrial = pFileInfo->fileSize < fileSize; - char* buf = taosMemoryMalloc(readSize + 5); - if (buf == NULL) { - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - return -1; - } - - int64_t offset; - offset = taosLSeekFile(pFile, -readSize, SEEK_END); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - char* found = NULL; + // search for the valid last WAL entry, e.g. block by block while (1) { + offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE); + ASSERT(offset <= end); + readSize = end - offset; + capacity = readSize + sizeof(magic); + + int64_t limit = WAL_RECOV_SIZE_LIMIT; + if (limit < readSize) { + wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64 + ", end:%" PRId64 ", file:%s", + pWal->cfg.vgId, limit, offset, end, fnameStr); + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + goto _err; + } + + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (readSize != taosReadFile(pFile, buf, readSize)) { + wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + readSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + char* candidate = NULL; char* haystack = buf; - char* candidate; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { - // read and validate - SWalCkHead* logContent = (SWalCkHead*)candidate; - if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { - found = candidate; + + while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + // validate head + int64_t len = readSize - (candidate - buf); + if (len < walCkHeadSz) { + break; } + SWalCkHead* logContent = (SWalCkHead*)candidate; + if (walValidHeadCksum(logContent) != 0) { + wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } + } + + // validate body + int64_t size = walCkHeadSz + logContent->head.bodyLen; + if (len < size) { + int64_t extraSize = size - len; + if (capacity < readSize + extraSize + sizeof(magic)) { + capacity += extraSize; + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + } + int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + offset); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", + pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + } + if (walValidBodyCksum(logContent) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } + } + + // found one + found = candidate; haystack = candidate + 1; } + if (found || offset == 0) break; - offset = TMIN(0, offset - readSize + sizeof(uint64_t)); - int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET); - ASSERT(offset == offset2); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } -#if 0 - if (found == buf) { - SWalCkHead* logContent = (SWalCkHead*)found; - if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { - // file has to be deleted - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } - } -#endif + + // go backwards, e.g. by at most one WAL scan buf size + end = offset + walCkHeadSz - 1; + firstTrial = false; } - if (found == NULL) { - // file corrupted, no complete log - // TODO delete and search in previous files - /*ASSERT(0);*/ - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; + // determine end of last entry + SWalCkHead* lastEntry = (SWalCkHead*)found; + int64_t retVer = -1; + int64_t lastEntryBeginOffset = 0; + int64_t lastEntryEndOffset = 0; + + if (lastEntry == NULL) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + } else { + retVer = lastEntry->head.version; + lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf); + lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; } // truncate file - SWalCkHead* lastEntry = (SWalCkHead*)found; - int64_t retVer = lastEntry->head.version; - int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf); - int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; if (lastEntryEndOffset != fileSize) { - wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, + wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, fileSize); - taosFtruncateFile(pFile, lastEntryEndOffset); - ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; - pWal->totSize -= (fileSize - lastEntryEndOffset); + if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { + wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (taosFsyncFile(pFile) < 0) { + wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } + pFileInfo->fileSize = lastEntryEndOffset; taosCloseFile(&pFile); taosMemoryFree(buf); - return retVer; + +_err: + taosCloseFile(&pFile); + taosMemoryFree(buf); + return -1; +} + +static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { + int metaFileNum = taosArrayGetSize(metaLogList); + int actualFileNum = taosArrayGetSize(actualLogList); + int j = 0; + + // both of the lists in asc order + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); + while (j < metaFileNum) { + SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); + ASSERT(pMetaInfo != NULL); + if (pMetaInfo->firstVer < pLogInfo->firstVer) { + j++; + } else if (pMetaInfo->firstVer == pLogInfo->firstVer) { + (*pLogInfo) = *pMetaInfo; + j++; + break; + } else { + break; + } + } + } + + taosArrayClear(metaLogList); + + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i); + taosArrayPush(metaLogList, pFileInfo); + } +} + +void walAlignVersions(SWal* pWal) { + if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { + wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, + pWal->vers.firstVer, pWal->vers.snapshotVer); + pWal->vers.firstVer = pWal->vers.snapshotVer + 1; + } + if (pWal->vers.lastVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.lastVer, pWal->vers.snapshotVer); + pWal->vers.lastVer = pWal->vers.snapshotVer; + } + if (pWal->vers.commitVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.commitVer, pWal->vers.snapshotVer); + pWal->vers.commitVer = pWal->vers.snapshotVer; + } + if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.appliedVer, pWal->vers.snapshotVer); + pWal->vers.appliedVer = pWal->vers.snapshotVer; + } + + pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer); + pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); } int walCheckAndRepairMeta(SWal* pWal) { @@ -150,7 +288,6 @@ int walCheckAndRepairMeta(SWal* pWal) { const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; - bool fixed = false; regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); @@ -184,222 +321,237 @@ int walCheckAndRepairMeta(SWal* pWal) { taosArraySort(actualLog, compareWalFileInfo); - int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); - int actualFileNum = taosArrayGetSize(actualLog); + int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); + int actualFileNum = taosArrayGetSize(actualLog); + int64_t firstVerPrev = pWal->vers.firstVer; + int64_t lastVerPrev = pWal->vers.lastVer; + int64_t totSize = 0; + bool updateMeta = (metaFileNum != actualFileNum); -#if 0 - for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) { - SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo); + // rebuild meta of file info + walRebuildFileInfoSet(pWal->fileInfoSet, actualLog); + taosArrayDestroy(actualLog); + + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + ASSERT(sz == actualFileNum); + + // scan and determine the lastVer + int32_t fileIdx = sz; + + while (--fileIdx >= 0) { char fnameStr[WAL_FILE_LEN]; + int64_t fileSize = 0; + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fileSize = 0; - taosStatFile(fnameStr, &fileSize, NULL); - if (fileSize == 0) { + int32_t code = taosStatFile(fnameStr, &fileSize, NULL); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); + return -1; + } + + ASSERT(pFileInfo->firstVer >= 0); + + if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { + totSize += pFileInfo->fileSize; + continue; + } + updateMeta = true; + + int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx); + if (lastVer < 0) { + if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { + wError("failed to scan wal last ver since %s", terrstr()); + return -1; + } + ASSERT(pFileInfo->fileSize == 0); + // remove the empty wal log, and its idx taosRemoveFile(fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); taosRemoveFile(fnameStr); - taosArrayPop(pLogInfoArray); - } else { - break; + // remove its meta entry + taosArrayRemove(pWal->fileInfoSet, fileIdx); + continue; } + + // update lastVer + pFileInfo->lastVer = lastVer; + totSize += pFileInfo->fileSize; } - actualFileNum = taosArrayGetSize(pLogInfoArray); -#endif - - { - int32_t i = 0, j = 0; - while (i < actualFileNum && j < metaFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j); - if (pActualFile->firstVer < pMetaFile->firstVer) { - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - i++; - } else if (pActualFile->firstVer > pMetaFile->firstVer) { - taosArrayRemove(pWal->fileInfoSet, j); - metaFileNum--; - } else { - i++; - j++; - } - } - if (i == actualFileNum && j == metaFileNum) { - if (j > 0) { - SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1); - int64_t fsize = 0; - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastInfo->firstVer, fNameStr); - taosStatFile(fNameStr, &fsize, NULL); - if (pLastInfo->fileSize != fsize) { - fixed = true; - pLastInfo->fileSize = fsize; - pLastInfo->lastVer = walScanLogGetLastVer(pWal); - } - } - } else { - fixed = true; - while (i < actualFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosStatFile(fNameStr, &pActualFile->fileSize, NULL); - - if (pActualFile->fileSize == 0) { - ASSERT(i == actualFileNum - 1); - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - break; - } - - if (i < actualFileNum - 1) { - pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1; - taosArrayPush(pWal->fileInfoSet, pActualFile); - i++; - } else { - pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile); - pActualFile->lastVer = walScanLogGetLastVer(pWal); - if (pActualFile->lastVer == -1) { - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - taosArrayPop(pWal->fileInfoSet); - } - break; - } - } - } - } - -#if 0 - if (metaFileNum > actualFileNum) { - taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum); - } else if (metaFileNum < actualFileNum) { - for (int i = metaFileNum; i < actualFileNum; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i); - taosArrayPush(pWal->fileInfoSet, pFileInfo); - } - } -#endif - - taosArrayDestroy(actualLog); - + // reset vers info and so on actualFileNum = taosArrayGetSize(pWal->fileInfoSet); pWal->writeCur = actualFileNum - 1; - + pWal->totSize = totSize; + pWal->vers.lastVer = -1; if (actualFileNum > 0) { - int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer; - if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) { - fixed = true; - pWal->vers.lastVer = fLastVer; - } - int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; - if (fFirstVer != pWal->vers.firstVer) { - fixed = true; - pWal->vers.firstVer = fFirstVer; - } + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer; + } + (void)walAlignVersions(pWal); + + // update meta file + if (updateMeta) { + (void)walSaveMeta(pWal); + } + return 0; +} + +int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { + if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } - if (fixed) { - walSaveMeta(pWal); + if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (walValidHeadCksum(pCkHead) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + return -1; } return 0; } -int walCheckAndRepairIdx(SWal* pWal) { +int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - for (int32_t i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + ASSERT(fileIdx >= 0 && fileIdx < sz); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + char fLogNameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); + int64_t fileSize = 0; - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fsize; - TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); - if (pIdxFile == NULL) { - ASSERT(0); + if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) { + wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer); + if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { + return 0; + } + + // start to repair + int64_t offset = fileSize - fileSize % sizeof(SWalIdxEntry); + TdFilePtr pLogFile = NULL; + TdFilePtr pIdxFile = NULL; + SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)}; + SWalCkHead ckHead; + memset(&ckHead, 0, sizeof(ckHead)); + ckHead.head.version = idxEntry.ver; + + pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); + if (pIdxFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); + if (pLogFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); + goto _err; + } + + // determine the last valid entry end, i.e. offset + while ((offset -= sizeof(SWalIdxEntry)) >= 0) { + if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr()); - return -1; + goto _err; } - taosFStatFile(pIdxFile, &fsize, NULL); - if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { - taosCloseFile(&pIdxFile); + if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (idxEntry.ver > pFileInfo->lastVer) { continue; } - int32_t left = fsize % sizeof(SWalIdxEntry); - int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); - if (left != 0) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr, - offset, fsize); - } - offset -= sizeof(SWalIdxEntry); - - SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer}; - while (1) { - if (offset < 0) { - taosLSeekFile(pIdxFile, 0, SEEK_SET); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - break; - } - taosLSeekFile(pIdxFile, offset, SEEK_SET); - int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld", - pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset); - offset -= sizeof(SWalIdxEntry); - } else { - break; - } + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "", + pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver); + continue; } - if (idxEntry.ver < pFileInfo->lastVer) { - char fLogNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); - TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); - if (pLogFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); - return -1; - } - while (idxEntry.ver < pFileInfo->lastVer) { - if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset, - terrstr()); - return -1; - } - SWalCkHead ckHead; - taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead)); - if (idxEntry.ver != ckHead.head.version) { - // todo truncate this idx also - taosCloseFile(&pLogFile); - wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId, - idxEntry.offset, idxEntry.ver, ckHead.head.version); - return -1; - } - idxEntry.ver = ckHead.head.version + 1; - idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen; - wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - } - taosCloseFile(&pLogFile); + if (idxEntry.ver == ckHead.head.version) { + break; + } + } + offset += sizeof(SWalIdxEntry); + + // ftruncate idx file + if (offset < fileSize) { + if (taosFtruncateFile(pIdxFile, offset) < 0) { + wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + strerror(errno), offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + // rebuild idx file + if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + while (idxEntry.ver < pFileInfo->lastVer) { + ASSERT(idxEntry.ver == ckHead.head.version); + + idxEntry.ver += 1; + idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; + + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), + idxEntry.offset, fLogNameStr); + goto _err; + } + wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); + if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { + wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + } + + if (taosFsyncFile(pIdxFile) < 0) { + wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return 0; + +_err: + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return -1; +} + +int walCheckAndRepairIdx(SWal* pWal) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + int32_t fileIdx = sz; + while (--fileIdx >= 0) { + if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) { + wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx); + return -1; } - taosCloseFile(&pIdxFile); } return 0; } @@ -492,14 +644,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; pRoot = cJSON_Parse(bytes); + if (!pRoot) goto _err; pMeta = cJSON_GetObjectItem(pRoot, "meta"); + if (!pMeta) goto _err; pField = cJSON_GetObjectItem(pMeta, "firstVer"); + if (!pField) goto _err; pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); + if (!pField) goto _err; pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); + if (!pField) goto _err; pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); + if (!pField) goto _err; pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); @@ -509,17 +667,23 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { taosArrayEnsureCap(pArray, sz); SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + if (!pInfoJson) goto _err; SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); + if (!pField) goto _err; pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); + if (!pField) goto _err; pInfo->lastVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "createTs"); + if (!pField) goto _err; pInfo->createTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "closeTs"); + if (!pField) goto _err; pInfo->closeTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "fileSize"); + if (!pField) goto _err; pInfo->fileSize = atoll(cJSON_GetStringValue(pField)); } taosArraySetSize(pArray, sz); @@ -527,6 +691,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pWal->writeCur = sz - 1; cJSON_Delete(pRoot); return 0; + +_err: + cJSON_Delete(pRoot); + return -1; } static int walFindCurMetaVer(SWal* pWal) { @@ -562,22 +730,63 @@ static int walFindCurMetaVer(SWal* pWal) { int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; - walBuildMetaName(pWal, metaVer + 1, fnameStr); - TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE); - if (pMetaFile == NULL) { + char tmpFnameStr[WAL_FILE_LEN]; + int n; + + // fsync the idx and log file at first to ensure validity of meta + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + + if (taosFsyncFile(pWal->pLogFile) < 0) { + wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + // flush to a tmpfile + n = walBuildTmpMetaName(pWal, tmpFnameStr); + ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); + + TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pMetaFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); if (len != taosWriteFile(pMetaFile, serialized, len)) { - // TODO:clean file - - taosCloseFile(&pMetaFile); - taosRemoveFile(fnameStr); - return -1; + wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile(pMetaFile) < 0) { + wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&pMetaFile) < 0) { + wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // rename it + n = walBuildMetaName(pWal, metaVer + 1, fnameStr); + ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); + + if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { + wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - taosCloseFile(&pMetaFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -585,6 +794,11 @@ int walSaveMeta(SWal* pWal) { } taosMemoryFree(serialized); return 0; + +_err: + taosCloseFile(&pMetaFile); + taosMemoryFree(serialized); + return -1; } int walLoadMeta(SWal* pWal) { @@ -626,6 +840,10 @@ int walLoadMeta(SWal* pWal) { } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); + if (code < 0) { + wError("failed to deserialize wal meta. file:%s", fnameStr); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + } taosCloseFile(&pFile); taosMemoryFree(buf); return code; @@ -636,6 +854,5 @@ int walRemoveMeta(SWal* pWal) { if (metaVer == -1) return 0; char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); - taosRemoveFile(fnameStr); - return 0; + return taosRemoveFile(fnameStr); } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 7974f3e32e..29058cada1 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pWal); + return NULL; + } + // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); @@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { tstrncpy(pWal->path, path, sizeof(pWal->path)); if (taosMkDir(pWal->path) != 0) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init ref pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pWal->pRefHash == NULL) { - taosMemoryFree(pWal); - return NULL; + wError("failed to init hash since %s", tstrerror(terrno)); + goto _err; } // open meta @@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init status @@ -131,46 +134,37 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; - if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { - taosArrayDestroy(pWal->fileInfoSet); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; - } - - pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if (pWal->refId < 0) { - taosHashCleanup(pWal->pRefHash); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - taosMemoryFree(pWal); - return NULL; - } - - walLoadMeta(pWal); + // load meta + (void)walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; } if (walCheckAndRepairIdx(pWal) < 0) { wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; + } + + // add ref + pWal->refId = taosAddRef(tsWal.refSetId, pWal); + if (pWal->refId < 0) { + wError("failed to add ref for Wal since %s", tstrerror(terrno)); + goto _err; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); - return pWal; + +_err: + taosArrayDestroy(pWal->fileInfoSet); + taosHashCleanup(pWal->pRefHash); + taosThreadMutexDestroy(&pWal->mutex); + taosMemoryFree(pWal); + pWal = NULL; + return NULL; } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { @@ -195,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { taosThreadMutexLock(&pWal->mutex); + (void)walSaveMeta(pWal); taosCloseFile(&pWal->pLogFile); pWal->pLogFile = NULL; taosCloseFile(&pWal->pIdxFile); pWal->pIdxFile = NULL; - walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; taosHashCleanup(pWal->pRefHash); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c4f1a81eeb..179d809c84 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); + if (pRet == NULL) { + wError("failed to find WAL log file with ver:%lld", ver); + terrno = TSDB_CODE_WAL_INVALID_VER; + return -1; + } if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner if (walReadChangeFile(pReader, pRet->firstVer) < 0) { @@ -472,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - ASSERT(0); + wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -505,6 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } + wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -514,7 +521,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { pReader->pHead->head.version, ver); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -528,7 +534,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 1196914dae..4b75db52b7 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { + code = taosFsyncFile(pWal->pLogFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { } } if (pWal->pIdxFile != NULL) { + code = taosFsyncFile(pWal->pIdxFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 628b432446..91fa49fce0 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -42,10 +42,22 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, i); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); } } walRemoveMeta(pWal); @@ -209,10 +221,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosCloseFile(&pIdxFile); taosCloseFile(&pLogFile); - taosFsyncFile(pWal->pLogFile); - taosFsyncFile(pWal->pIdxFile); - - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } // unlock taosThreadMutexUnlock(&pWal->mutex); @@ -384,7 +398,11 @@ int32_t walRollImpl(SWal *pWal) { pWal->lastRollSeq = walGetSeq(); - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + goto END; + } END: return code; @@ -540,6 +558,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), + strerror(errno)); + } wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index f0442c6fd1..616ab7875d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -87,11 +87,17 @@ void osUpdate() { void osCleanup() {} -bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } +bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; } -bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; } +bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } -bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; } +bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; } + +bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; } + +bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; } + +bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 8f026f6890..75a963d5c7 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -228,7 +228,7 @@ void taosPrintBackTrace() { void taosPrintBackTrace() { return; } #endif -void *taosMemoryMalloc(int32_t size) { +void *taosMemoryMalloc(int64_t size) { #ifdef USE_TD_MEMORY void *tmp = malloc(size + sizeof(TdMemoryInfo)); if (tmp == NULL) return NULL; @@ -244,7 +244,7 @@ void *taosMemoryMalloc(int32_t size) { #endif } -void *taosMemoryCalloc(int32_t num, int32_t size) { +void *taosMemoryCalloc(int64_t num, int64_t size) { #ifdef USE_TD_MEMORY int32_t memorySize = num * size; char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); @@ -261,7 +261,7 @@ void *taosMemoryCalloc(int32_t num, int32_t size) { #endif } -void *taosMemoryRealloc(void *ptr, int32_t size) { +void *taosMemoryRealloc(void *ptr, int64_t size) { #ifdef USE_TD_MEMORY if (ptr == NULL) return taosMemoryMalloc(size); @@ -318,7 +318,7 @@ void taosMemoryFree(void *ptr) { #endif } -int32_t taosMemorySize(void *ptr) { +int64_t taosMemorySize(void *ptr) { if (ptr == NULL) return 0; #ifdef USE_TD_MEMORY diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 7e1506c140..f3ccbb0aac 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -19,12 +19,12 @@ #include "taos.h" #include "taoserror.h" -#define UNIT_NUM_BITS 64 -#define UNIT_ADDR_NUM_BITS 6 +#define UNIT_NUM_BITS 64ULL +#define UNIT_ADDR_NUM_BITS 6ULL static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); uint64_t old = buf[unitIndex]; buf[unitIndex] |= mask; return buf[unitIndex] != old; @@ -32,7 +32,7 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index eb13a08be4..9a117c6eb4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -447,12 +447,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Wal unexpected generic error") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 65f1bac60a..cbbf53d8f1 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -219,7 +219,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { while (temp != pTree->NIL) { y = temp; - int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp)); + int32_t c = pTree->cmprFn(z, temp); if (c < 0) { temp = temp->left; } else if (c > 0) { @@ -232,7 +232,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { if (y == pTree->NIL) { pTree->root = z; - } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) { + } else if (pTree->cmprFn(z, y) < 0) { y->left = z; } else { y->right = z; @@ -245,10 +245,10 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { tRBTreePutFix(pTree, z); // update min/max node - if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) { + if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min, z) > 0) { pTree->min = z; } - if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) { + if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max, z) < 0) { pTree->max = z; } pTree->n++; @@ -309,11 +309,11 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { return pNode; } -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { - int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode)); + int32_t c = pTree->cmprFn(pKeyNode, pNode); if (c < 0) { pNode = pNode->left; diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 4ce668e6ba..d93e9fc569 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -145,7 +145,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it tSkipListWLock(pSkipList); void *pData = iterate(iter); - if (pData == NULL) return; + if (pData == NULL) { + tSkipListUnlock(pSkipList); + return; + } // backward to put the first data hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 0a835ec564..c68375894c 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -32,6 +32,10 @@ class TDTestCase: if "community" in selfPath: projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] else: projPath = selfPath[: selfPath.find("tests")] @@ -55,11 +59,10 @@ class TDTestCase: tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") - tdSql.query("select count(*) from (select distinct(tbname) from db.stb)") - tdSql.checkData(0, 0, 10) + tdSql.query("show db.tables") + tdSql.checkRows(10) tdSql.query("select count(*) from db.stb") - if len(tdSql.queryResult): - tdLog.exit("query result is %d" % len(tdSql.queryResult)) + tdSql.checkData(0, 0, 100) def stop(self): tdSql.close() diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json index d4b2aae2fb..da22ef75e2 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json @@ -20,8 +20,10 @@ "super_tables": [{ "name": "stb", "childtable_prefix": "stb_", + "childtable_count": 10, + "insert_rows": 10, "columns": [{"type": "INT"}], "tags": [{"type": "INT"}] }] }] -} \ No newline at end of file +} diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh new file mode 100755 index 0000000000..457c9eae20 --- /dev/null +++ b/tests/script/coverage_test.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +branch= +if [ x$1 != x ];then + branch=$1 + echo "Testing branch: $branch" +else + echo "Please enter branch name as a parameter" + exit 1 +fi + +today=`date +"%Y%m%d"` +TDENGINE_DIR=/root/pxiao/TDengine +TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDengine { + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + + # reset counter + lcov -d . --zerocounters + + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + fi + + git reset --hard + git checkout -- . + git checkout $branch + git clean -dfx + git pull + git submodule update --init --recursive -f + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + echo "rebuild.." + LOCAL_COMMIT=`git rev-parse --short @` + + rm -rf * + if [ "$branch" == "3.0" ]; then + echo "3.0 =============" + cmake -DCOVER=true -DBUILD_TEST=true .. + else + cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null + fi + make -j4 + make install +} + +function runGeneralCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + case=`echo $line | grep sim$ | awk '{print $NF}'` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTestNGCaseOneByOne { + while read -r line; do + if [[ $line =~ ^taostest* ]]; then + case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2` + yaml=`echo $line | cut -d' ' -f 2` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTest { + echo "run Test" + + if [ "$branch" == "3.0" ]; then + echo "start run unit test case ................" + echo " $TDENGINE_DIR/debug " + cd $TDENGINE_DIR/debug + ctest -j12 + echo "3.0 unit test done" + fi + + cd $TDENGINE_DIR/tests/script + + [ -d ../../sim ] && rm -rf ../../sim + [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT + + runGeneralCaseOneByOne jenkins/basic.txt + + sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT + + totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + sed "3G" $TDENGINE_COVERAGE_REPORT + + stopTaosd + echo "run TestNG cases" + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + cd $TDENGINE_DIR/../TestNG/cases + runTestNGCaseOneByOne ../scripts/cases.txt + echo "TestNG cases done" + + cd $TDENGINE_DIR/tests + rm -rf ../sim + /root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT + + + sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT + totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ "$totalPySuccess" -gt "0" ]; then + sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ -z $totalPyFailed ]; then + sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT + else + sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + + echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT + # Test Connector + stopTaosd + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + + cd $TDENGINE_DIR/src/connector/jdbc + mvn clean package > /dev/null 2>&1 + mvn test > jdbc-out.log 2>&1 + tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test C Demo + stopTaosd + $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null & + sleep 10 + yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test waltest + dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'` + walDir=`find $dataDir -name "wal"|head -n1` + echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT + echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT + if [ -n "$walDir" ]; then + yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + fi + + # run Unit Test + echo "Run Unit Test: utilTest, queryTest and cliTest" + #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!" + #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!" + #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!" + + stopTaosd + + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + # collect data + lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + if [ "$branch" == "3.0" ]; then + lcov --remove coverage.info \ + '*/contrib/*' '*/tests/*' '*/test/*'\ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\ + '*/tthread.c' '*/tversion.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + else + lcov --remove coverage.info \ + '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + fi + + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + + # push result to coveralls.io + echo "push result to coveralls.io" + /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + + #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT + #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT + cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function sendReport { + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasod end" +} + +function runTestRandomFail { + exec_random_fail_sh=$1 + default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh + [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1 + + dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py + default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py + [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1 + + runTest NoRandomFail +} + +WORK_DIR=/root/pxiao + +date >> $WORK_DIR/cron.log +echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log + +stopTaosd +buildTDengine + +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh + +runTest + +lcovFunc +#sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 82f73a4fdd..d8f4a36261 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -155,7 +155,7 @@ ./test.sh -f tsim/parser/select_with_tags.sim ./test.sh -f tsim/parser/selectResNum.sim ./test.sh -f tsim/parser/set_tag_vals.sim -./test.sh -f tsim/parser/single_row_in_tb.sim +# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim ./test.sh -f tsim/parser/sliding.sim ./test.sh -f tsim/parser/slimit_alter_tags.sim ./test.sh -f tsim/parser/slimit.sim @@ -194,7 +194,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -# TD-17919 ./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim # ---- show ---- @@ -352,6 +352,7 @@ ./test.sh -f tsim/scalar/in.sim ./test.sh -f tsim/scalar/scalar.sim ./test.sh -f tsim/scalar/filter.sim +./test.sh -f tsim/scalar/caseWhen.sim # ---- alter ---- ./test.sh -f tsim/alter/cached_schema_after_alter.sim diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim index 59156080c8..31fc276298 100644 --- a/tests/script/tsim/mnode/basic1.sim +++ b/tests/script/tsim/mnode/basic1.sim @@ -42,6 +42,7 @@ sql_error drop mnode on dnode 1 print =============== create mnode 2 sql create mnode on dnode 2 +print =============== create mnode 2 finished $x = 0 step2: $x = $x + 1 @@ -69,9 +70,10 @@ if $data(2)[2] != follower then goto step2 endi -sleep 2000 print ============ drop mnode 2 sql drop mnode on dnode 2 + +print ============ drop mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 1 then return -1 @@ -109,6 +111,8 @@ sleep 2000 print =============== create mnodes sql create mnode on dnode 2 + +print =============== create mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 2 then return -1 diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index a2b8aa2f96..6739a87b42 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -45,49 +45,12 @@ endi if $data(2)[4] != ready then goto step2 endi - -system sh/exec.sh -n dnode3 -s stop -sql_error create mnode on dnode 3 - -print =============== step3: select * from information_schema.ins_mnodes - -$x = 0 -step3: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi -sql select * from information_schema.ins_mnodes -x step3 -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] -print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] - -$leaderNum = 0 -if $data(1)[2] == leader then - $leaderNum = 1 -endi -if $data(2)[2] == leader then - $leaderNum = 1 -endi -if $leaderNum == 0 then - goto step3 -endi -if $data(3)[2] != offline then - goto step3 -endi -if $data(1)[3] != ready then - goto step3 -endi -if $data(2)[3] != ready then - goto step3 -endi -if $data(3)[3] != creating then - goto step3 +if $data(3)[4] != ready then + goto step2 endi -print =============== step4: start dnode3 -system sh/exec.sh -n dnode3 -s start +print =============== step4: create mnode 3 +sql create mnode on dnode 3 $x = 0 step4: @@ -159,12 +122,11 @@ endi if $data(2)[3] != ready then goto step5 endi -if $data(3)[3] != dropping then - goto step5 -endi print =============== step6: start dnode3 system sh/exec.sh -n dnode3 -s start +sql drop mnode on dnode 1 -x step60 +step60: $x = 0 step6: diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 387f38a717..16e8fa3dfa 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -232,12 +232,11 @@ endi if $leaderNum != 1 then goto step81 endi -if $data(1)[3] != dropping then - goto step81 -endi print =============== step9: start mnode1 and wait it dropped system sh/exec.sh -n dnode1 -s start +sql drop mnode on dnode 1 -x step90 +step90: $x = 0 step91: diff --git a/tests/script/tsim/scalar/caseWhen.sim b/tests/script/tsim/scalar/caseWhen.sim new file mode 100644 index 0000000000..f6b9c3ff08 --- /dev/null +++ b/tests/script/tsim/scalar/caseWhen.sim @@ -0,0 +1,1064 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== prepare data + +sql drop database if exists db1; +sql create database db1 vgroups 5; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(10), f3 bool) tags(t1 int, t2 bool, t3 binary(10)); +sql create table tba1 using sta tags(0, false, '0'); +sql create table tba2 using sta tags(1, true, '1'); +sql create table tba3 using sta tags(null, null, ''); +sql create table tba4 using sta tags(1, false, null); +sql create table tba5 using sta tags(3, true, 'aa'); +sql insert into tba1 values ('2022-09-26 15:15:01', 0, "a", false); +sql insert into tba1 values ('2022-09-26 15:15:02', 1, "0", true); +sql insert into tba1 values ('2022-09-26 15:15:03', 5, "5", false); +sql insert into tba1 values ('2022-09-26 15:15:04', null, null, null); +sql insert into tba2 values ('2022-09-27 15:15:01', 0, "a", false); +sql insert into tba2 values ('2022-09-27 15:15:02', 1, "0", true); +sql insert into tba2 values ('2022-09-27 15:15:03', 5, "5", false); +sql insert into tba2 values ('2022-09-27 15:15:04', null, null, null); +sql insert into tba3 values ('2022-09-28 15:15:01', 0, "a", false); +sql insert into tba3 values ('2022-09-28 15:15:02', 1, "0", true); +sql insert into tba3 values ('2022-09-28 15:15:03', 5, "5", false); +sql insert into tba3 values ('2022-09-28 15:15:04', null, null, null); +sql insert into tba4 values ('2022-09-29 15:15:01', 0, "a", false); +sql insert into tba4 values ('2022-09-29 15:15:02', 1, "0", true); +sql insert into tba4 values ('2022-09-29 15:15:03', 5, "5", false); +sql insert into tba4 values ('2022-09-29 15:15:04', null, null, null); +sql insert into tba5 values ('2022-09-30 15:15:01', 0, "a", false); +sql insert into tba5 values ('2022-09-30 15:15:02', 1, "0", true); +sql insert into tba5 values ('2022-09-30 15:15:03', 5, "5", false); +sql insert into tba5 values ('2022-09-30 15:15:04', null, null, null); + +print ======== case when xx + +sql select case when 3 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 4 then + return -1 +endi + +sql select case when 0 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when null then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 1 then 4+1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 5.000000000 then + return -1 +endi +if $data10 != 5.000000000 then + return -1 +endi +if $data20 != 5.000000000 then + return -1 +endi +if $data30 != 5.000000000 then + return -1 +endi + +sql select case when 1-1 then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 1+1 then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 1 then 1-1+2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 2.000000000 then + return -1 +endi +if $data30 != 2.000000000 then + return -1 +endi + +sql select case when 1 > 0 then 1 < 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when 1 > 2 then 1 < 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when abs(3) then abs(-1) end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when abs(1+1) then abs(-1)+abs(3) end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4.000000000 then + return -1 +endi +if $data10 != 4.000000000 then + return -1 +endi +if $data20 != 4.000000000 then + return -1 +endi +if $data30 != 4.000000000 then + return -1 +endi + +sql select case when 0 then 1 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 3 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 3 then + return -1 +endi + +sql select case when 0 then 1 when 1 then 0 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 0 then 1 when 1 then 0 when 2 then 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 'a' then 'b' when null then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when '2' then 'b' when null then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != b then + return -1 +endi +if $data10 != b then + return -1 +endi +if $data20 != b then + return -1 +endi +if $data30 != b then + return -1 +endi + +sql select case when 0 then 'b' else null end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 0 then 'b' else 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 2 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case when sum(2) then sum(2)-sum(1) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 4.000000000 then + return -1 +endi + +sql select case when sum(2) then abs(-2) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 2 then + return -1 +endi + +sql select case when ts then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != @22-09-26 15:15:01.000@ then + return -1 +endi +if $data10 != @22-09-26 15:15:02.000@ then + return -1 +endi +if $data20 != @22-09-26 15:15:03.000@ then + return -1 +endi +if $data30 != @22-09-26 15:15:04.000@ then + return -1 +endi + +sql select case when f1 then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != @22-09-26 15:15:02.000@ then + return -1 +endi +if $data20 != @22-09-26 15:15:03.000@ then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when f1 then f1 when f1 + 1 then f1 + 1 else f1 is null end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when f1 then 3 when ts then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1664176501000 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 1664176504000 then + return -1 +endi + +sql select case when 3 then f1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when f1 then 3 when 1 then 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case when sum(f1) then sum(f1)-abs(-1) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 5.000000000 then + return -1 +endi + +sql select case when sum(f1) then sum(f1)-abs(f1) end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != 0.000000000 then + return -1 +endi +if $data30 != 0.000000000 then + return -1 +endi + +sql select case when f1 then sum(f1) when f1 is not null then 9 else 8 end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != 8 then + return -1 +endi +if $data10 != 9 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 5 then + return -1 +endi + +sql select f1 from tba1 where f1 > case when f1 then 0 else 3 end; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 5 then + return -1 +endi + +sql select f1 from tba1 where ts > case when ts then ts end; +if $rows != 0 then + return -1 +endi + +sql select sum(f1),count(f1) from tba1 partition by case when f1 then f1 when 1 then 1 end; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +if $data11 != 1 then + return -1 +endi + +sql select case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end caseWhen, sum(f1),count(f1) from tba1 group by case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end order by caseWhen; +if $rows != 3 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data12 != 1 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data22 != 0 then + return -1 +endi + +sql select f1 from tba1 order by case when f1 <= 0 then 3 when f1 = 1 then 4 when f1 >= 3 then 2 else 1 end desc; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select cast(case f1 when f1 then f1 + 1 else f1 is null end as double) from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 6.000000000 then + return -1 +endi +if $data30 != 1.000000000 then + return -1 +endi + +sql select sum(case f1 when f1 then f1 + 1 else f1 is null end + 1) from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 14.000000000 then + return -1 +endi + +sql select case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end,sum(f1),count(f1) from tba1 state_window(case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end); +if $rows != 3 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data12 != 1 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data22 != 0 then + return -1 +endi + +sql select f1 from tba1 where case when case when f1 <= 0 then 3 when f1 = 1 then 4 when f1 >= 3 then 2 else 1 end > 2 then 1 else 0 end > 0; +if $rows != 2 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +sql select case when f1 is not null then case when f1 <= 0 then f1 else f1 * 10 end else -1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 10 then + return -1 +endi +if $data20 != 50 then + return -1 +endi +if $data30 != -1 then + return -1 +endi + + +print ======== case xx when xx + +sql select case 3 when 3 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 4 then + return -1 +endi + +sql select case 3 when 1 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case 3 when 1 then 4 else 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 2 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case 3 when null then 4 when '3' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case '3' when null then 4 when 3 then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case null when null then 4 when 3 then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case 3.0 when null then 4 when '3' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case f2 when 'a' then 4 when '0' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case f2 when f1 then f1 when f1 - 1 then f1 else 99 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 99 then + return -1 +endi + +sql select case cast(f2 as int) when 0 then f2 when f1 then 11 else ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != a then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 11 then + return -1 +endi +if $data30 != 1664176504 then + return -1 +endi + +sql select case f1 + 1 when 1 then 1 when 2 then 2 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 3 then + return -1 +endi + +sql select case f1 when sum(f1) then sum(f1)-abs(f1) end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != 0.000000000 then + return -1 +endi +if $data20 != 0.000000000 then + return -1 +endi +if $data30 != 0.000000000 then + return -1 +endi + +sql select f1, case sum(f1) when 1 then f1 + 99 when f1 then f1 -99 else f1 end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data11 != -99.000000000 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data21 != 100.000000000 then + return -1 +endi +if $data30 != 5 then + return -1 +endi +if $data31 != -94.000000000 then + return -1 +endi + +sql select case when 3 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when null then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1 then 4+1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1-1 then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1+1 then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when abs(3) then abs(-1) end from sta; +if $rows != 20 then + return -1 +endi +sql select case when abs(1+1) then abs(-1)+abs(3) end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 else 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 when 1 then 0 else 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 when 1 then 0 when 2 then 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 'a' then 'b' when null then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when '2' then 'b' when null then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 'b' else null end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 'b' else 2+abs(-2) end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 3 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 1 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 1 then 4 else 2 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when null then 4 when '3' then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case null when null then 4 when 3 then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3.0 when null then 4 when '3' then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select f2,case f2 when 'a' then 4 when '0' then 1 end from sta order by f2; +if $rows != 20 then + return -1 +endi +sql select f2,f1,case f2 when f1 then f1 when f1 - 1 then f1 else 99 end from sta order by f2; +if $rows != 20 then + return -1 +endi +sql select case cast(f2 as int) when 0 then f2 when f1 then 11 else ts end from sta; +if $rows != 20 then + return -1 +endi +sql select f1, case f1 + 1 when 1 then 1 when 2 then 2 else 3 end from sta order by f1; +if $rows != 20 then + return -1 +endi +sql select case f1 when sum(f1) then sum(f1)-abs(f1) end from sta group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != 0.000000000 then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi +sql select case sum(f1) when 1 then f1 + 99 when f1 then f1 -99 else f1 end from sta group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != -99.000000000 then + return -1 +endi +if $data20 != 1.000000000 then + return -1 +endi +if $data30 != 5.000000000 then + return -1 +endi + +sql select distinct tbname, case t1 when t2 then t1 else t1 + 100 end from sta order by tbname; +if $rows != 5 then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data31 != 101 then + return -1 +endi +if $data41 != 103 then + return -1 +endi + +sql_error select case when sum(f1) then sum(f1)-abs(f1) end from tba1; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 6fa7e39335..252ef10b1a 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -62,7 +62,16 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data -sql select * from c1 + +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select * from c1 -x steps + sql select * from stb sql select * from stb_1 sql select ts, c1, c2, c3 from c1 diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 11a387ed4d..89b6cadef8 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -158,9 +158,15 @@ print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v -sleep 1000 +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select avg(tbcol) as c from stb -x steps -sql select avg(tbcol) as c from stb sql select avg(tbcol) as c from stb where ts <= 1601481840000 sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 sql select avg(tbcol) as c from stb interval(1m) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py new file mode 100644 index 0000000000..d3a55ee0a6 --- /dev/null +++ b/tests/system-test/1-insert/alter_database.py @@ -0,0 +1,58 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.buffer_boundary = [3,4097,8193,12289,16384] + self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + # pages_boundary >= 64 + self.pages_boundary = [64,128,512] + self.pages_error = [self.pages_boundary[0]-1] + def alter_buffer(self): + tdSql.execute('create database db') + for buffer in self.buffer_boundary: + tdSql.execute(f'alter database db buffer {buffer}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.execute('drop database db') + tdSql.execute('create database db vgroups 10') + for buffer in self.buffer_error: + tdSql.error(f'alter database db buffer {buffer}') + tdSql.execute('drop database db') + + def alter_pages(self): + tdSql.execute('create database db') + for pages in self.pages_boundary: + tdSql.execute(f'alter database db pages {pages}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + self.pages_error.append(tdSql.queryResult[0][10]) + for pages in self.pages_error: + tdSql.error(f'alter database db pages {pages}') + tdSql.execute('drop database db') + + def run(self): + tdSql.error('create database db1 vgroups 10 buffer 12289') + self.alter_buffer() + self.alter_pages() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py new file mode 100644 index 0000000000..d18cb0260e --- /dev/null +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -0,0 +1,77 @@ +import taos +import sys +import time +import socket +import os +import threading +from util.common import * + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.setsql = TDSetSql() + self.rowNum = 10 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + } + self.error_topic = ['avg','count','spread','stddev','sum','hyperloglog'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def wrong_topic(self): + tdSql.prepare() + tdSql.execute('use db') + stbname = f'db.{tdCom.getLongName(5, "letters")}' + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + tdSql.execute(f"create table {stbname}_tb1 using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_tb1',self.rowNum) + for column in self.column_dict.keys(): + for func in self.error_topic: + if func.lower() != 'count' and column.lower() != 'ts': + tdSql.error(f'create topic tpn as select {func}({column}) from {stbname}') + elif func.lower() == 'count' : + tdSql.error(f'create topic tpn as select {func}(*) from {stbname}') + for column in self.column_dict.keys(): + if column.lower() != 'ts': + tdSql.error(f'create topic tpn as select apercentile({column},50) from {stbname}') + tdSql.error(f'create topic tpn as select leastquares({column},1,1) from {stbname}_tb1') + tdSql.error(f'create topic tpn as select HISTOGRAM({column},user_input,[1,3,5,7],0) from {stbname}') + tdSql.error(f'create topic tpn as select percentile({column},1) from {stbname}_tb1') + pass + def run(self): + self.wrong_topic() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index a613f11267..de95fef148 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -52,7 +52,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1c6aef6286..39adfc9d53 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -#python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py @@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 0-others/compatibility.py - +python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py @@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -# unstable python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3 @@ -248,8 +248,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 @@ -279,7 +279,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 - +python3 ./test.py -f 7-tmq/create_wrong_topic.py python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py @@ -302,7 +302,7 @@ python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqCheckData1.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py +#python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqAlterSchema.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py diff --git a/tools/scripts/codeFormat.sh b/tools/scripts/codeFormat.sh old mode 100644 new mode 100755 index 8707dcdabc..cc7db5c723 --- a/tools/scripts/codeFormat.sh +++ b/tools/scripts/codeFormat.sh @@ -12,25 +12,7 @@ FORMAT_DIR_LIST=( "${PRJ_ROOT_DIR}/source/os" "${PRJ_ROOT_DIR}/source/util" "${PRJ_ROOT_DIR}/source/common" - "${PRJ_ROOT_DIR}/source/libs/cache" - "${PRJ_ROOT_DIR}/source/libs/catalog" - "${PRJ_ROOT_DIR}/source/libs/command" - "${PRJ_ROOT_DIR}/source/libs/executor" - "${PRJ_ROOT_DIR}/source/libs/function" - "${PRJ_ROOT_DIR}/source/libs/index" - "${PRJ_ROOT_DIR}/source/libs/monitor" - "${PRJ_ROOT_DIR}/source/libs/nodes" - # "${PRJ_ROOT_DIR}/source/libs/parser" - "${PRJ_ROOT_DIR}/source/libs/planner" - "${PRJ_ROOT_DIR}/source/libs/qcom" - "${PRJ_ROOT_DIR}/source/libs/qworker" - "${PRJ_ROOT_DIR}/source/libs/scalar" - "${PRJ_ROOT_DIR}/source/libs/stream" - "${PRJ_ROOT_DIR}/source/libs/sync" - "${PRJ_ROOT_DIR}/source/libs/tdb" - "${PRJ_ROOT_DIR}/source/libs/tfs" - "${PRJ_ROOT_DIR}/source/libs/transport" - "${PRJ_ROOT_DIR}/source/libs/wal" + "${PRJ_ROOT_DIR}/source/libs" "${PRJ_ROOT_DIR}/source/client/inc" "${PRJ_ROOT_DIR}/source/client/src" "${PRJ_ROOT_DIR}/source/client/test" @@ -45,7 +27,7 @@ EXCLUDE_FILE_LIST=( ) for d in ${FORMAT_DIR_LIST[@]}; do - for f in $(find $d -type f -regex '.*\.\(cpp\|hpp\|c\|h\)'); do + for f in $(find $d -type f -not -name '*sql.c' -regex '.*\.\(cpp\|hpp\|c\|h\)'); do ${FORMAT_BIN} -i $f done done diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 8aafd747af..46eaf6cff1 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -325,24 +325,23 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { - printf(" **************************** How To Use TAB Key ********************************\n"); - printf(" * TDengine Command Line supports pressing TAB key to complete word, *\n"); - printf(" * including database name, table name, function name and keywords. *\n"); - printf(" * Press TAB key anywhere, you'll get surprise. *\n"); - printf(" * KEYBOARD SHORTCUT: *\n"); - printf(" * [ TAB ] ...... Complete the word or show help if no input *\n"); - printf(" * [ Ctrl + A ] ...... move cursor to [A]head of line *\n"); - printf(" * [ Ctrl + E ] ...... move cursor to [E]nd of line *\n"); - printf(" * [ Ctrl + W ] ...... move cursor to line of middle *\n"); - printf(" * [ Ctrl + L ] ...... clean screen *\n"); - printf(" * [ Ctrl + K ] ...... clean after cursor *\n"); - printf(" * [ Ctrl + U ] ...... clean before cursor *\n"); - printf(" * *\n"); + printf(" ****************************** Tab Completion **********************************\n"); + printf(" * The TDengine CLI supports tab completion for a variety of items, *\n"); + printf(" * including database names, table names, function names and keywords. *\n"); + printf(" * The full list of shortcut keys is as follows: *\n"); + printf(" * [ TAB ] ...... complete the current word *\n"); + printf(" * ...... if used on a blank line, display all valid commands *\n"); + printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n"); + printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n"); + printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n"); + printf(" * [ Ctrl + L ] ...... clear the entire screen *\n"); + printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n"); + printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n"); printf(" **********************************************************************************\n\n"); } void showHelp() { - printf("\nThe following are supported commands for TDengine Command Line:"); + printf("\nThe TDengine CLI supports the following commands:"); printf( "\n\ ----- A ----- \n\ diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index fd0ea60323..37d00c0a7c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) { printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); // call auto tab - callbackAutoTab(command, pSql, false); + callbackAutoTab(command, NULL, false); } printf("\r\n"); @@ -998,7 +998,9 @@ void *shellCancelHandler(void *arg) { shell.stop_query = true; } else { #endif - taos_kill_query(shell.conn); + if (shell.conn) { + taos_kill_query(shell.conn); + } #ifdef WEBSOCKET } #endif diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index 2f1ee12d54..b310281119 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -243,8 +243,8 @@ void enumAllWords(STireNode** nodes, char* prefix, SMatch* match) { continue; } else { // combine word string - memset(word, 0, sizeof(word)); - strcpy(word, prefix); + memset(word, 0, tListLen(word)); + strncpy(word, prefix, len); word[len] = FIRST_ASCII + i; // append current char // chain middle node @@ -315,8 +315,7 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { } } - // return - return; + taosMemoryFree(root); } SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) { diff --git a/tools/taosws-rs b/tools/taosws-rs new file mode 160000 index 0000000000..7a94ffab45 --- /dev/null +++ b/tools/taosws-rs @@ -0,0 +1 @@ +Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 56ba622a9c..b6d8d75ba0 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -119,7 +119,7 @@ int smlProcess_json1_Test() { " \"dc\": \"lga\"" " }" " }" - "]"}; + "]",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -159,7 +159,7 @@ int smlProcess_json2_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -227,7 +227,7 @@ int smlProcess_json3_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -286,7 +286,7 @@ int smlProcess_json4_Test() { " \"t9\": false," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index a043aa7a6d..d36fe0855a 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -155,7 +155,7 @@ static void printHelp() { printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s%s%d\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); printf("%s%s\n", indent, "-b"); @@ -238,7 +238,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); + taosFprintfFile(g_fp, " expect rows: %" PRIx64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt); } char tmpString[128]; @@ -263,11 +263,11 @@ void parseArgument(int32_t argc, char* argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); + tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName)); } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.cdbName, argv[++i]); + tstrncpy(g_stConfInfo.cdbName, argv[++i], sizeof(g_stConfInfo.cdbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { @@ -279,9 +279,9 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-e") == 0) { g_stConfInfo.useSnapshot = atol(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { - char tmpBuf[56]; - strcpy(tmpBuf, argv[++i]); - sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + char tmpBuf[56] = {0}; + tstrncpy(tmpBuf, argv[++i], sizeof(tmpBuf)); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); } else if (strcmp(argv[i], "-x") == 0) { g_stConfInfo.numOfThread = atol(argv[++i]); } else if (strcmp(argv[i], "-l") == 0) { @@ -294,6 +294,10 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.producerRate = atol(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { g_stConfInfo.payloadLen = atol(argv[++i]); + if(g_stConfInfo.payloadLen <= 0 || g_stConfInfo.payloadLen > 1024 * 1024 * 1024){ + pError("%s calloc size is too large: %s %s", GREEN, argv[++i], NC); + exit(-1); + } } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -354,8 +358,8 @@ void ltrim(char* str) { int queryDB(TAOS* taos, char* command) { int retryCnt = 10; - int code; - TAOS_RES* pRes; + int code = 0; + TAOS_RES* pRes = NULL; while (retryCnt--) { pRes = taos_query(taos, command); @@ -363,10 +367,11 @@ int queryDB(TAOS* taos, char* command) { if (code != 0) { taosSsleep(1); taos_free_result(pRes); + pRes = NULL; continue; } taos_free_result(pRes); - return 0; + return 0; } pError("failed to reason:%s, sql: %s", tstrerror(code), command); @@ -418,7 +423,7 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { char sqlStr[1100] = {0}; if (strlen(buf) > 1024) { - taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", (int)strlen(buf)); taosCloseFile(&g_fp); return -1; } @@ -592,7 +597,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -644,7 +649,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -960,7 +965,7 @@ void parseConsumeInfo() { ltrim(pstr); char* ret = strchr(pstr, ch); memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1, sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey])); // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; @@ -1268,25 +1273,26 @@ void* ombProduceThreadFunc(void* param) { for (int i = 0; i < batchPerTblTimes; ++i) { uint32_t msgsOfSql = g_stConfInfo.batchSize; if ((i == batchPerTblTimes - 1) && (0 != remainder)) { - msgsOfSql = remainder; + msgsOfSql = remainder; } int len = 0; len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); for (int j = 0; j < msgsOfSql; j++) { - int64_t timeStamp = taosGetTimestampNs(); - len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); sendMsgs++; pInfo->totalProduceMsgs++; } - totalMsgLen += len; + totalMsgLen += len; pInfo->totalMsgsLen += len; - int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); if (affectedRows < 0) { taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; + pInfo->taos = NULL; + taosMemoryFree(sqlBuf); + return NULL; } affectedRowsTotal += affectedRows; @@ -1322,6 +1328,7 @@ void* ombProduceThreadFunc(void* param) { printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); taos_close(pInfo->taos); pInfo->taos = NULL; + taosMemoryFree(sqlBuf); return NULL; } diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 3a598ba98b..6540fdac4c 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -663,7 +663,7 @@ void initLogFile() { int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { if(strcmp(argv[i], "-c") == 0){ - strcpy(g_conf.dir, argv[++i]); + tstrncpy(g_conf.dir, argv[++i], sizeof(g_conf.dir)); }else if(strcmp(argv[i], "-s") == 0){ g_conf.snapShot = true; }else if(strcmp(argv[i], "-d") == 0){